
    (HJj                     j   d dl mZ d dlmZmZmZmZ d dlmZ	 d dl
mZ d dlmZ ddlmZ ddlmZmZmZ ddlmZ e G d	 d
e             Z G d dej.                        Z G d dej.                        Z G d dej.                        Z G d dej.                        Z G d dej.                        Zy)    )	dataclass)AnyDictOptionalUnionN)shard_linear   )swiglu)BaseModelArgscreate_attention_maskscaled_dot_product_attention)initialize_ropec                       e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed	<   eed
<   eed<   eed<   dZe	e
eeeef   f      ed<   y)	ModelArgs
model_typehidden_sizenum_hidden_layersintermediate_sizenum_attention_headsrms_norm_eps
vocab_sizenum_key_value_headsmax_position_embeddings
rope_thetahead_dimtie_word_embeddingsNrope_scaling)__name__
__module____qualname__str__annotations__intfloatboolr   r   r   r        ]/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx_lm/models/qwen3.pyr   r      si    OO  M;?L(4U5#:%6 678?r'   r   c            	            e Zd Zdef fdZ	 	 ddej                  deej                     dee   dej                  fdZ	 xZ
S )		Attentionargsc                    t         |           |j                  }|j                  x| _        }|j
                  J |j
                  x| _        }|j                  }|dz  | _        t        j                  |||z  d      | _        t        j                  |||z  d      | _        t        j                  |||z  d      | _        t        j                  ||z  |d      | _        t        j                  ||j                         | _        t        j                  ||j                         | _        t'        ||j(                  d|j*                  |j,                        | _        y )Ng      Fbiaseps)basetraditionalscaling_configr   )super__init__r   r   n_headsr   
n_kv_headsr   scalennLinearq_projk_projv_projo_projRMSNormr   q_normk_normr   r   r   r   rope)selfr+   dimr6   r7   r   	__class__s         r(   r5   zAttention.__init__!   s"   !%!9!99w''333'+'?'??*==t^
iiWx%7eDiiZ(%:GiiZ(%:Gii( 2CeDjjt/@/@Ajjt/@/@A#,,$($@$@
	r'   xmaskcachereturnc                 n   |j                   \  }}}| j                  |      | j                  |      | j                  |      }	}}| j	                  |j                  ||| j                  d            j                  dddd      }| j                  |j                  ||| j                  d            j                  dddd      }|	j                  ||| j                  d      j                  dddd      }	|P| j                  ||j                        }| j                  ||j                        }|j                  ||	      \  }}	n"| j                  |      }| j                  |      }t        |||	|| j                  |      }
|
j                  dddd      j                  ||d      }
| j                  |
      S )Nr      r	      )offset)rH   r8   rG   )shaper;   r<   r=   r@   reshaper6   	transposerA   r7   rB   rN   update_and_fetchr   r8   r>   )rC   rF   rG   rH   BLDquerieskeysvaluesoutputs              r(   __call__zAttention.__call__;   s    ''1a $AAAv++gooaDLL"EFPPq!Q
 {{4<<1doorBCMMq!Q
 1door:DDQ1aPiii=G99T%,,97D 11$?LD&ii(G99T?D-T6djjt
 !!!Q1-55aB?{{6""r'   NNr   r   r    r   r5   mxarrayr   r   rZ   __classcell__rE   s   @r(   r*   r*       sW    
Y 
: $(#	#88# rxx # }	#
 
#r'   r*   c                   >     e Zd Z fdZdej
                  fdZ xZS )MLPc                     t         |           t        j                  ||d      | _        t        j                  ||d      | _        t        j                  ||d      | _        y NFr-   )r4   r5   r9   r:   	gate_proj	down_projup_proj)rC   rD   
hidden_dimrE   s      r(   r5   zMLP.__init__]   sJ    3
?:s?yyju=r'   rI   c                 t    | j                  t        | j                  |      | j                  |                  S N)rf   r
   re   rg   )rC   rF   s     r(   rZ   zMLP.__call__c   s)    ~~fT^^A%6QHIIr'   )r   r   r    r5   r]   r^   rZ   r_   r`   s   @r(   rb   rb   \   s    >JRXX Jr'   rb   c            	            e Zd Zdef fdZ	 	 ddej                  deej                     dee   dej                  fdZ	 xZ
S )	TransformerBlockr+   c                    t         |           |j                  | _        |j                  | _        t	        |      | _        t        |j                  |j                        | _        t        j                  |j                  |j                        | _        t        j                  |j                  |j                        | _        || _        y )Nr/   )r4   r5   r   r   r*   	self_attnrb   r   mlpr9   r?   r   input_layernormpost_attention_layernormr+   rC   r+   rE   s     r(   r5   zTransformerBlock.__init__h   s    #'#;#; ++"4t'')?)?@!zz$*:*:@Q@QR(*

$"3"3)
% 	r'   rF   rG   rH   rI   c                     | j                  | j                  |      ||      }||z   }| j                  | j                  |            }||z   }|S rj   )rn   rp   ro   rq   )rC   rF   rG   rH   rhouts          r(   rZ   zTransformerBlock.__call__t   sQ     NN4//2D%@EHHT22156!e
r'   r[   r\   r`   s   @r(   rl   rl   g   sW    
Y 
 $(#	
88
 rxx 
 }	

 

r'   rl   c                   h     e Zd Zdef fdZ	 	 ddej                  deej                     fdZ xZ	S )
Qwen3Modelr+   c                    t         |           || _        |j                  | _        |j                  | _        | j                  dkD  sJ t        j                  |j                  |j                        | _        t        |j                        D cg c]  }t        |       c}| _        t        j                  |j                  |j                        | _        y c c}w )Nr   )r+   r/   )r4   r5   r+   r   r   r9   	Embeddingr   embed_tokensrangerl   layersr?   r   norm)rC   r+   _rE   s      r(   r5   zQwen3Model.__init__   s    	//!%!7!7"""LL$:J:JK16t7M7M1N
1NA$'1N
 JJt//T5F5FG	
s   Cinputsinput_embeddingsc                     ||}n| j                  |      }|d gt        | j                        z  }t        ||d         }t	        | j                  |      D ]  \  }} ||||      } | j                  |      S )Nr   )r{   lenr}   r   zipr~   )rC   r   rH   r   ru   rG   layercs           r(   rZ   zQwen3Model.__call__   s     ' A!!&)A=FS--E$Qa1DKK/HE1aq!A 0 yy|r'   r[   )
r   r   r    r   r5   r]   r^   r   rZ   r_   r`   s   @r(   rx   rx      s=    
HY 
H /3	 #288,	r'   rx   c                        e Zd Zdef fdZ	 	 d
dej                  deej                     fdZd Z	ddeej                  j                     fdZed	        Z xZS )Modelr+   c                     t         |           || _        |j                  | _        t	        |      | _        |j                  s2t        j                  |j                  |j                  d      | _        y y rd   )r4   r5   r+   r   rx   modelr   r9   r:   r   r   lm_headrr   s     r(   r5   zModel.__init__   sX    	//%
''99T%5%5tUSDL (r'   r   r   c                     | j                  |||      }| j                  j                  r'| j                   j                  j	                  |      }|S | j                  |      }|S rj   )r   r+   r   r{   	as_linearr   )rC   r   rH   r   rv   s        r(   rZ   zModel.__call__   sZ     jj(8999((**))33C8C 
 ,,s#C
r'   c                 V    | j                   j                  r|j                  dd        |S )Nzlm_head.weight)r+   r   pop)rC   weightss     r(   sanitizezModel.sanitize   s#    99((KK($/r'   groupc                    |xs t         j                  j                         }|j                         }| j                  j
                  D ]  }t        |j                  j                  d|      |j                  _        t        |j                  j                  d|      |j                  _	        t        |j                  j                  d|      |j                  _
        t        |j                  j                  d|      |j                  _        |j                  xj                  |z  c_        |j                  xj                  |z  c_        t        |j                  j                  d|      |j                  _        t        |j                  j                   d|      |j                  _        t        |j                  j"                  d|      |j                  _         y )Nzall-to-sharded)r   zsharded-to-all)r]   distributedinitsizer   r}   r   rn   r;   r<   r=   r>   r6   r7   ro   re   rf   rg   )rC   r   Nr   s       r(   shardzModel.shard   se   .,,.JJLZZ&&E%1&&(8&EOO" &2&&(8&EOO" &2&&(8&EOO" &2&&(8&EOO" OO##)#OO&&1,& #/		##%5U#EII #/		##%5U#EII !-		!!#35!EII1 'r'   c                 .    | j                   j                  S rj   )r   r}   )rC   s    r(   r}   zModel.layers   s    zz   r'   r[   rj   )r   r   r    r   r5   r]   r^   r   rZ   r   r   Groupr   propertyr}   r_   r`   s   @r(   r   r      sq    TY T /3	 #288,	
8BNN$8$89 > ! !r'   r   )dataclassesr   typingr   r   r   r   mlx.corecorer]   mlx.nnr9   mlx.nn.layers.distributedr   activationsr
   r1   r   r   r   
rope_utilsr   r   Moduler*   rb   rl   rx   r   r&   r'   r(   <module>r      s    " - -   2  T T ' @ @ @ 9#		 9#xJ")) Jryy 4 D<!BII <!r'   