
    (HJj?                    h   d dl mZ d dlmZ d dlmZ d dlmZmZm	Z	m
Z
mZmZ d dlmZ d dlmZ d dlmZ ddlmZ dd	lmZmZmZmZ dd
lmZmZ ddlmZ ddl m!Z! ddl"m#Z# e G d de             Z$ eejJ                  d      d        Z& G d dejN                        Z( G d dejN                        Z) G d dejN                        Z* G d dejN                        Z+ G d dejN                        Z, G d dejN                        Z- G d d ejN                        Z. G d! d"ejN                        Z/y)#    )annotations)	dataclass)partial)AnyDictListOptionalTupleUnionN)sum_gradients   )swiglu)BaseModelArgscreate_attention_maskcreate_ssm_maskscaled_dot_product_attention)ArraysCacheKVCache)gated_delta_update)initialize_rope)	SwitchGLUc                  :   e Zd ZU ded<   ded<   ded<   ded<   ded<   ded<   ded	<   ded
<   ded<   ded<   ded<   ded<   ded<   ded<   ded<   ded<   ded<   ded<   ded<   ded<   ded<   ded<   ded<   dZded<   dZded<   dZded <   d!Zd"ed#<   d$Zded%<   y!)&	ModelArgsstr
model_typeinthidden_sizenum_hidden_layersintermediate_sizenum_attention_headslinear_num_value_headslinear_num_key_headslinear_key_head_dimlinear_value_head_dimlinear_conv_kernel_dimnum_expertsnum_experts_per_tokdecoder_sparse_stepshared_expert_intermediate_sizez	List[int]mlp_only_layersmoe_intermediate_sizefloatrms_norm_eps
vocab_sizenum_key_value_heads
rope_thetapartial_rotary_factormax_position_embeddingshead_dimFboolnorm_topk_probtie_word_embeddingsattention_biasNz&Optional[Dict[str, Union[float, str]]]rope_scaling   full_attention_interval)	__name__
__module____qualname____annotations__r5   r6   r7   r8   r:        b/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx_lm/models/qwen3_next.pyr   r      s    O%((O    M ND  %% ND ;?L8?#$S$r@   r   T)	shapelessc                    t        j                  |j                  t        j                              }|j                  t        j                        }||z  j                  | j
                        S N)nnsiluastypemxfloat32dtype)hgatexs      rA   _precise_swiglurN   :   sH    774;;rzz*+D	A1HQWW%%r@   c                  8     e Zd Zdd fdZ	 d	 	 	 	 	 ddZ xZS )Qwen3NextRMSNormGatedc                d    t         |           || _        t        j                  |      | _        y rD   )super__init__epsrH   onesweight)selfr   rT   	__class__s      rA   rS   zQwen3NextRMSNormGated.__init__B   s%    ggk*r@   c                    t         j                  j                  || j                  | j                        }|t        |||      S |j                  |j                        S rD   )rH   fastrms_normrV   rT   rN   rG   rJ   )rW   hidden_statesrL   rM   s       rA   __call__zQwen3NextRMSNormGated.__call__G   sN     GG]DKKB"=$::88M//00r@   )ư>)r   r   rT   r,   rD   )r\   mx.arrayrL   zmx.array | Nonereturnr_   r;   r<   r=   rS   r]   __classcell__rX   s   @rA   rP   rP   A   s,    + @D1%1-<1	1r@   rP   c                  <     e Zd Zd fdZ	 	 d	 	 	 	 	 	 	 ddZ xZS )Qwen3NextAttentionc                \   t         |           |j                  | _        |j                  | _        |j                  | _        | j                  dz  | _        t        j                  |j                  | j                  | j                  z  dz  |j                        | _
        t        j                  |j                  | j                  | j                  z  |j                        | _        t        j                  |j                  | j                  | j                  z  |j                        | _        t        j                  | j                  | j                  z  |j                  |j                        | _        t        j                  | j                  |j                        | _        t        j                  | j                  |j                        | _        t%        t'        | j                  |j(                  z        |j*                  d|j,                  |j.                        | _        y )N         biasrT   F)basetraditionalscaling_configr2   )rR   rS   r/   r    r3   scalerE   Linearr   r7   q_projk_projv_projo_projRMSNormr-   q_normk_normr   r   r1   r0   r8   r2   roperW   argsrX   s     rA   rS   zQwen3NextAttention.__init__R   s   #'#;#; #'#;#; ]]D(
ii$$t}}4q8$$

 ii$$t}}4$$

 ii$$t}}4$$

 ii$$t}}4$$
 jjD4E4EFjjD4E4EF# : ::;,,$($@$@
	r@   c                   |j                   \  }}}| j                  |      }t        j                  |j	                  ||| j
                  d      dd      \  }}	|	j	                  ||d      }	| j                  |      | j                  |      }}
| j                  |      j                  dddd      }| j                  |
j	                  ||| j                  d            j                  dddd      }
|j	                  ||| j                  d      j                  dddd      }|P| j                  ||j                        }| j                  |
|j                        }
|j                  |
|      \  }
}n"| j                  |      }| j                  |
      }
t        ||
||| j                   |      }|j                  dddd      j	                  ||d      }| j#                  |t        j$                  |	      z        S )	Nrh   axisr   r      )offset)cachero   mask)shaperq   rH   splitreshaper    rr   rs   rv   	transposerw   r/   rx   r   update_and_fetchr   ro   rt   sigmoid)rW   rM   r   r   BLDq_proj_outputqueriesrL   keysvaluesoutputs                rA   r]   zQwen3NextAttention.__call__y   s    ''1aA!!!Q(@(@"Eqr
 ||Aq"%{{1~t{{1~f++g&00Aq!<{{4<<1d.F.FKLVVq!Q
 1d&>&>CMMq!Q
 iii=G99T%,,97D 11$?LD&ii(G99T?D-T6djjt
 !!!Q1-55aB?{{6BJJt$4455r@   rz   r   NNrM   r_   r   Optional[mx.array]r   Optional[Any]r`   r_   ra   rc   s   @rA   re   re   Q   s<    %
T $(#	%6%6 !%6 	%6
 
%6r@   re   c                  &     e Zd Z fdZddZ xZS )Qwen3NextMLPc                    t         |           t        j                  ||d      | _        t        j                  ||d      | _        t        j                  ||d      | _        y NFri   )rR   rS   rE   rp   	gate_proj	down_projup_proj)rW   dim
hidden_dimrX   s      rA   rS   zQwen3NextMLP.__init__   sJ    3
?:s?yyju=r@   c                t    | j                  t        | j                  |      | j                  |                  S rD   )r   r   r   r   )rW   rM   s     rA   r]   zQwen3NextMLP.__call__   s)    ~~fT^^A%6QHIIr@   )r`   r_   ra   rc   s   @rA   r   r      s    >Jr@   r   c                  P     e Zd Zd fdZ	 	 	 	 	 	 ddZ	 	 d	 	 	 	 	 	 	 ddZ xZS )Qwen3NextGatedDeltaNetc                R   t         |           |j                  | _        |j                  | _        |j
                  | _        |j                  | _        |j                  | _
        | j                  | j                  z  | _        | j                  | j                  z  | _        | j                  | j                  z  dk7  r&t        d| j                   d| j                   d      |j                  | _        |j                   | _        | j                  dz  | j                  z   | _        t'        j(                  | j$                  | j$                  d| j                  | j$                  d      | _        t'        j,                  | j                  | j                  dz  | j                  dz  z   d      | _        t'        j,                  | j                  | j                  dz  d      | _        t3        j4                  | j                        | _        t2        j8                  j;                  dd	| j                  f
      }t3        j<                  |      | _        tA        | j                  | j"                        | _!        t'        j,                  | j                  | j                  d      | _"        y )Nr   znum_v_heads (z$) must be divisible by num_k_heads ()rh   F)in_channelsout_channelsrj   kernel_sizegroupspaddingri      )lowhighr   rk   )#rR   rS   r   r!   num_v_headsr"   num_k_headsr#   
head_k_dimr$   
head_v_dimkey_dim	value_dim
ValueErrorr%   conv_kernel_sizer-   layer_norm_epsilonconv_dimrE   Conv1dconv1drp   in_proj_qkvz
in_proj_barH   rU   dt_biasrandomuniformlogA_logrP   normout_proj)rW   configArX   s      rA   rS   zQwen3NextGatedDeltaNet.__init__   s   !--!88!66 44 66)9)994+;+;;d...!3 0 011UVZVfVfUgghi  !' = ="("5"5q(4>>9ii--==
 IIdllQ.!1CC%
 ))D$4$4d6F6F6JQVWwwt//0II!"T5E5E4GHVVAY
)$//t?V?VW			$..$2B2BOr@   c                   | j                   | j                  | j                  | j                  f\  }}}} |j                  g |j
                  d d |d } |j                  g |j
                  d d |d }t        j                  ||d|z  d|z  ||z  |z  z   gd      \  }}}	}
t        j                  |||z  gd      \  }}|| |	j                  g |	j
                  d d d|  |
j                  g |
j
                  d d d|  |j                  g |j
                  d d |  |j                  g |j
                  d d | fS )Nr|   rh   r}   )r   r   r   r   r   r   rH   r   )rW   
mixed_qkvzmixed_bankdnnvdvqkvzbas                rA   fix_query_key_value_orderingz3Qwen3NextGatedDeltaNet.fix_query_key_value_ordering   sx    OOOO	
BB (Z''G)9)9#2)>GGBG
#8##AX^^CR%8A"AbAXXj2q2vq2vb27M*NUWX
1axx28*261AII+qwwr{+B++AII+qwwr{+B++AII'qwwr{'B'AII'qwwr{'B'
 	
r@   c                   |j                   \  }}}| j                  | j                  |      | j                  |            \  }}}	}
}}||d   |d   }n;t	        j
                  || j                  dz
  | j                  f|j                        }t	        j                  |j                  ||d      |j                  ||d      |	j                  ||d      gd      }|t	        j                  |d   |d      }t	        j                  ||gd      }|| j                  dz
  }|j                  _t	        j                  |j                  d|      }|d d d f   t	        j                  |      z   d   }t	        j                  ||d      |d<   n%t	        j                   |d d | d d d f         |d<   t#        j$                  | j'                  |            }t)        t	        j*                  || j,                  d| j,                  z  gd      | j.                  | j.                  | j0                  g| j2                  | j2                  | j4                  g      D cg c]  \  }}}|j                  ||||       c}}}\  }}}	|r|d   nd }|j                   d   dz  }|dz  t        j6                  j9                  |d d	      z  }|t        j6                  j9                  |d d	      z  }t;        |||	||| j<                  | j>                  ||| j@                   

      \  }}|||d<   |jC                  |       | jE                  ||
      }| jG                  |j                  ||d            S c c}}}w )Nr   r   )rJ   r|   r}   .Nrh   rg   r^   )
use_kernel)$r   r   r   r   rH   zerosr   r   rJ   concatenater   wherelengthscliparangetake_along_axis
contiguousrE   rF   r   zipr   r   r   r   r   r   rZ   r[   r   r   r   trainingadvancer   r   )rW   inputsr   r   r   S_r   r   r   r   r   r   
conv_state	mixed_qkv
conv_inputn_keepends	positionsconv_outtrK   dstate	inv_scaleouts                             rA   r]   zQwen3NextGatedDeltaNet.__call__   s.    ,,1a<<f%tv'>
1aAq q!5qJD))A-t}}=llJ
 NNYYq!R !))Aq""5qyyAr7JKRT
	 i)Q?I^^Z$;!D
**Q.F}}(wwu}}a3!!T']RYYv->>	J	--j)!La==AwxN)CDa774;;z23 DLL!dll2B#CRH!!4#3#3T5E5EF$//4??C
1a IIaAq!
1a "atGGBK4'	\RWW--at<<((D$77'JJLL==(

U E!HMM!iiQ}}S[[Ar233A
s   M))r   r   )r   r_   r   r_   r`   r_   r   )r   r_   r   r   r   r   r`   r_   )r;   r<   r=   rS   r   r]   rb   rc   s   @rA   r   r      s_    'PR
"
.6
	
2 $(#	E4E4 !E4 	E4
 
E4r@   r   c                  0     e Zd Zd fdZ	 	 	 	 ddZ xZS )Qwen3NextSparseMoeBlockc                   t         |           |j                  }|j                  }|j                  }|j
                  | _        |j                  x| _        }|j                  | _        t        j                  ||d      | _        t        |||      | _        t        ||      | _        t        j                  |dd      | _        d | _        y )NFri   r   )rR   rS   r   r+   r)   r5   r&   r'   top_krE   rp   rL   r   
switch_mlpr   shared_expertshared_expert_gatesharding_group)rW   rz   r   r   r)   r&   rX   s         rA   rS   z Qwen3NextSparseMoeBlock.__init__5  s     66*.*N*N'"11)-)9)99;--
IIc;U;	#C):KH)#/NO"$))C"?"r@   c                   | j                    t        | j                         |      }| j                  |      }t        j                  |dd      }| j
                  }t        j                  || d      d| d f   }t        j                  ||d      }| j                  r||j                  dd      z  }| j                  ||      }||d   z  j                  d	      }| j                  |      }t        j                  | j                  |            |z  }||z   }| j                   +t        j                  j                  || j                   
      }|S )Nr|   T)r~   precise)kthr~   .r}   )r~   keepdimsr   )group)r   r   rL   rH   softmaxr   argpartitionr   r5   sumr   r   r   r   distributedall_sum)rW   rM   gatesr   indsscoresyshared_ys           rA   r]   z Qwen3NextSparseMoeBlock.__call__G  s8    *2d11215A		!

5r48JJu1"26sQBCx@##E4b9fjjb4j@@FOOAt$	""''R'0%%a(::d55a89HDL*&&q0C0C&DAr@   r   )rM   r_   r`   r_   ra   rc   s   @rA   r   r   4  s    #$ 
r@   r   c                  <     e Zd Zd fdZ	 	 d	 	 	 	 	 	 	 ddZ xZS )Qwen3NextDecoderLayerc                @   t         |           |dz   |j                  z  dk7  | _        | j                  rt	        |      | _        nt        |      | _        t        j                  |j                  |j                        | _        t        j                  |j                  |j                        | _        ||j                  vr5|j                  dkD  r&|dz   |j                   z  dk(  rt#        |      | _        y t'        |j                  |j(                        | _        y )Nr   r   rk   )rR   rS   r:   	is_linearr   linear_attnre   	self_attnrE   ru   r   r-   input_layernormpost_attention_layernormr*   r&   r(   r   mlpr   r   )rW   rz   	layer_idxrX   s      rA   rS   zQwen3NextDecoderLayer.__init__f  s    #a-4+G+GG1L>>5d;D/5DN!zz$*:*:@Q@QR(*

$"3"3)
% T111q i!mt7O7O%OST%T.t4DH#D$4$4d6L6LMDHr@   c                    | j                   r#| j                  | j                  |      ||      }n"| j                  | j                  |      ||      }||z   }|| j	                  | j                  |            z   }|S rD   )r	  r
  r  r  r  r  )rW   rM   r   r   rrK   r   s          rA   r]   zQwen3NextDecoderLayer.__call__y  ss     >>  !5!5a!8$FAt33A6eDAE$((488;<<
r@   )rz   r   r  r   r   r   ra   rc   s   @rA   r  r  e  s<    N, $(#	 ! 	
 
r@   r  c                  6     e Zd Zd fdZ	 d	 	 	 	 	 ddZ xZS )Qwen3NextModelc                   t         |           t        j                  |j                  |j
                        | _        t        |j                        D cg c]  }t        ||       c}| _
        t        j                  |j
                  |j                        | _        d| _        |j                  dz
  | _        y c c}w )N)rz   r  rk   r   r   )rR   rS   rE   	Embeddingr.   r   embed_tokensranger   r  layersru   r-   r   ssm_idxr:   fa_idx)rW   rz   irX   s      rA   rS   zQwen3NextModel.__init__  s    LL$:J:JK 4112
2 "tq92
 JJt//T5F5FG	22Q6
s   B>c                R   | j                  |      }|d gt        | j                        z  }t        ||| j                           }t        ||| j                           }t        | j                  |      D ]   \  }}|j                  r|n|} ||||      }" | j                  |      S )N)r   r   )
r  lenr  r   r  r   r  r   r	  r   )	rW   r   r   r\   fa_maskssm_masklayercr   s	            rA   r]   zQwen3NextModel.__call__  s    
 ))&1=FS--E'uT[[7IJ"=%2EFDKK/HE1$8GD!-d!DM 0 yy''r@   r   rD   r   r_   r   r   r`   r_   ra   rc   s   @rA   r  r    s.    	7  $(( ( 
	(r@   r  c                  b     e Zd Zd fdZ	 d	 	 	 	 	 d	dZed        Zd Zd Zed        Z	 xZ
S )
Modelc                    t         |           || _        |j                  | _        t	        |      | _        |j                  s2t        j                  |j                  |j                  d      | _        y y r   )rR   rS   rz   r   r  modelr6   rE   rp   r   r.   lm_headry   s     rA   rS   zModel.__init__  sX    	//#D)
''99T%5%5tUSDL (r@   c                    | j                  ||      }| j                  j                  r'| j                   j                  j	                  |      }|S | j                  |      }|S rD   )r&  rz   r6   r  	as_linearr'  )rW   r   r   r   s       rA   r]   zModel.__call__  sW    
 jj'99((**))33C8C 
 ,,s#C
r@   c                .    | j                   j                  S rD   )r&  r  )rW   s    rA   r  zModel.layers  s    zz   r@   c                    | j                   D cg c]$  }|j                  rt        d      n	t               & c}S c c}w )Nrh   )size)r  r	  r   r   )rW   ls     rA   
make_cachezModel.make_cache  s0    KO;;W;aq{{#	A;WWWs   );c                   d|vr|S |j                         D ci c]  \  }}d|vs|| }}}| j                  j                  r|j                  dd        t	        | j                  j
                        D ]q  }d| d}dD ]d  }t	        | j                  j                        D cg c]  }|j                  | d| d| d	       }}t        j                  |      || d
| d	<   f s d}	|j                         D ]]  \  }
dv r'|
j                  d   dk7  r|
j                  dd      |<   t        fd|	D              sF|
j                  dk(  sV|
dz   |<   _ |S c c}}w c c}w )Nz+model.layers.0.mlp.experts.0.up_proj.weightzmtp.zlm_head.weightzmodel.layers.z.mlp)r   r   r   z	.experts..z.weightz.switch_mlp.)z.input_layernorm.weightz .post_attention_layernorm.weightzmodel.norm.weightz.q_norm.weightz.k_norm.weightzconv1d.weightr|   r   rh   c              3  @   K   | ]  }j                  |        y wrD   endswith).0sfxr   s     rA   	<genexpr>z!Model.sanitize.<locals>.<genexpr>  s     8is1::c?is   g      ?)itemsrz   r6   popr  r   r&   rH   stackr   moveaxisanyndim)rW   weightskeyvaluer-  prefixneto_join	norm_keysr   r   s              @rA   sanitizezModel.sanitize  sw   8GN07U*#u6QTCT3:U99((KK($/tyy223A$QCt,F: #499#8#899 KK6()A3as' BC9   >@XXg=N6(,qc9: ; 4
	 MMODAq!#q(8ZZ1-
8i8866Q;!"SGAJ $ 7 Vs   EE&!E%c                    d }|S )Nc                R    | j                  d      s| j                  d      rdddS y)Nzmlp.gater   @      )
group_sizebitsTr2  )pathr   s     rA   	predicatez(Model.quant_predicate.<locals>.predicate  s(    }}Z(DMM:N,O&(!44r@   r?   )rW   rM  s     rA   quant_predicatezModel.quant_predicate  s    	
 r@   r   rD   r"  )r;   r<   r=   rS   r]   propertyr  r.  rE  rN  rb   rc   s   @rA   r$  r$    sc    T  $

 
 
	
 ! !X@  r@   r$  )0
__future__r   dataclassesr   	functoolsr   typingr   r   r   r	   r
   r   mlx.corecorerH   mlx.nnrE   mlx.nn.layers.distributedr   activationsr   rl   r   r   r   r   r   r   r   gated_deltar   
rope_utilsr   switch_layersr   r   compilerN   ModulerP   re   r   r   r   r  r  r$  r?   r@   rA   <module>r^     s   # !  : :   3   ( + ' $ % % %> 	t$& %&1BII 1 M6 M6`J299 JE4RYY E4P.bii .b BII  F(RYY (@CBII Cr@   