
    (HJj                        d dl mZ d dlmZ d dlmZmZmZmZ d dl	m
Z d dlmZ ddlmZmZmZ e G d de             Z eej&                  d	
      d        Z G d dej*                        Z G d dej.                        Z G d dej.                        Z G d dej.                        Z G d dej.                        Z G d dej.                        Zy)    )	dataclass)partial)AnyDictOptionalUnionN   )BaseModelArgscreate_attention_maskscaled_dot_product_attentionc                      e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed	<   d
Zee   ed<   d
Z	ee   ed<   dZ
eed<   dZeed<   dZeed<   dZeed<   dZeed<   d
Zeeeeeef   f      ed<   dZeed<   d Zy
)	ModelArgs
model_typehidden_size
hidden_actnum_hidden_layersintermediate_sizenum_attention_headsnorm_eps
vocab_sizenum_key_value_headsNhead_dimmax_position_embeddingsFattention_biasmlp_biasg      ?partial_rotary_factorg     @
rope_thetarope_traditionalrope_scalingtie_word_embeddingsc                     | j                   rnd| j                   vrt        d      | j                   j                  d      xs | j                   j                  d      }|t        d      |dvrt        d      y y )Nfactorz"rope_scaling must contain 'factor'type	rope_typez6rope_scaling must contain either 'type' or 'rope_type')linearz4rope_scaling 'type' currently only supports 'linear')r   
ValueErrorget)selfr$   s     `/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx_lm/models/nemotron.py__post_init__zModelArgs.__post_init__"   s    t000 #EGG))--f5 9J9J9N9N:I   L  
* !WXX +     )__name__
__module____qualname__str__annotations__intfloatr   r   r   r   boolr   r   r   r   r   r   r   r    r*    r+   r)   r   r      s    OOOO"Hhsm"-1Xc]1 ND Hd#&5&J"d";?L(4U5#:%6 678? %%Yr+   r   T)	shapelessc                 H    t        j                  |       j                         S N)nnrelusquare)xs    r)   relu_squaredr<   1   s    771:r+   c                       e Zd Zd Zy)NemotronLayerNorm1Pc                     d| v r| j                   dz   nd }d| v r| j                  nd }t        j                  j	                  |||| j
                        S )Nweightr	   bias)r@   rA   mxfast
layer_normeps)r(   r;   r@   rA   s       r)   __call__zNemotronLayerNorm1P.__call__7   sH    $,$4q$"dNtyyww!!!VT488<<r+   N)r,   r-   r.   rF   r4   r+   r)   r>   r>   6   s    =r+   r>   c            	            e Zd Zdef fdZ	 	 ddej                  deej                     dee   dej                  fdZ	 xZ
S )		Attentionargsc                 j   t         |           |j                  }|j                  x| _        }|j
                  x| _        }|j                  xs |j                  |z  x| _        }|j                  | _        |dz  | _	        t        |d      r|j                  }nd}t        j                  |||z  |      | _        t        j                  |||z  |      | _        t        j                  |||z  |      | _        t        j                  ||z  ||      | _        d}|j$                  rC|j$                  d   dk(  r1t'        |j$                  d   t(              sJ d	|j$                  d   z  }t        j*                  t-        | j                  | j                  z        |j.                  |
      | _        y )Ng      r   FrA   g      ?r#   r%   r"   r	   )basescale)super__init__r   r   n_headsr   
n_kv_headsr   r   rM   hasattrr   r8   Linearq_projk_projv_projo_projr   
isinstancer2   RoPEr1   r   rope)	r(   rI   dimrP   rQ   r   r   
rope_scale	__class__s	           r)   rO   zAttention.__init__>   sv   !%!9!99w'+'?'??*#'==#OD4D4D4OO%)%?%?"t^
4)*!00N"NiiWx%7nMiiZ(%:PiiZ(%:Pii( 2CnM
!2!26!:h!Fd//95AAAT..x88JGG**T]]:;
	r+   r;   maskcachereturnc                 2   |j                   \  }}}| j                  |      | j                  |      | j                  |      }	}}|j	                  ||| j
                  d      j                  dddd      }|j	                  ||| j                  d      j                  dddd      }|	j	                  ||| j                  d      j                  dddd      }	|P| j                  ||j                        }| j                  ||j                        }|j                  ||	      \  }}	n"| j                  |      }| j                  |      }t        |||	|| j                  |      }
|
j                  dddd      j	                  ||d      }
| j                  |
      S )Nr      r	      )offset)r_   rM   r^   )shaperT   rU   rV   reshaperP   	transposerQ   rZ   re   update_and_fetchr   rM   rW   )r(   r;   r^   r_   BL_querieskeysvaluesoutputs              r)   rF   zAttention.__call__]   si    ''1a $AAAv //!Qb9CCAq!QO||Aq$//26@@Aq!L1door:DDQ1aPiii=G99T%,,97D 11$?LD&ii(G99T?D-T6djjt
 !!!Q1-55aB?{{6""r+   NNr,   r-   r.   r   rO   rB   arrayr   r   rF   __classcell__r]   s   @r)   rH   rH   =   sX    
Y 
D $(#	#88# rxx # }	#
 
#r+   rH   c                   D     e Zd Zdef fdZdej                  fdZ xZS )MLPrI   c                     t         |           |j                  }|j                  }|j                  }t        j                  |||      | _        t        j                  |||      | _        y )NrK   )	rN   rO   r   r   r   r8   rS   	down_projup_proj)r(   rI   r[   
hidden_dimr   r]   s        r)   rO   zMLP.__init__|   sU    ++
==:sByyjx@r+   r`   c                 T    | j                  t        | j                  |                  S r7   )ry   r<   rz   )r(   r;   s     r)   rF   zMLP.__call__   s    ~~l4<<?;<<r+   	r,   r-   r.   r   rO   rB   rs   rF   rt   ru   s   @r)   rw   rw   {   s!    AY A=RXX =r+   rw   c            	            e Zd Zdef fdZ	 	 ddej                  deej                     dee   dej                  fdZ	 xZ
S )	TransformerBlockrI   c                 >   t         |           |j                  | _        |j                  | _        t	        |      | _        t        |      | _        t        |j                  |j                        | _
        t        |j                  |j                        | _        y )NrE   )rN   rO   r   r   rH   	self_attnrw   mlpr>   r   input_layernormpost_attention_layernormr(   rI   r]   s     r)   rO   zTransformerBlock.__init__   su    #'#;#; ++"4t9243C3CW(;$--)
%r+   r;   r^   r_   r`   c                     | j                  | j                  |      ||      }||z   }| j                  | j                  |            }||z   }|S r7   )r   r   r   r   )r(   r;   r^   r_   rhouts          r)   rF   zTransformerBlock.__call__   sQ     NN4//2D%@EHHT22156!e
r+   rq   rr   ru   s   @r)   r   r      sW    	
Y 	
 $(#	
88
 rxx 
 }	

 

r+   r   c                   H     e Zd Zdef fdZ	 ddej                  fdZ xZS )NemotronModelrI   c                    t         |           || _        |j                  | _        |j                  | _        | j                  dkD  sJ t        j                  |j                  |j                        | _        t        |j                        D cg c]  }t        |       c}| _        t        |j                  |j                        | _        y c c}w )Nr   )rI   r   )rN   rO   rI   r   r   r8   	Embeddingr   embed_tokensranger   layersr>   r   norm)r(   rI   rl   r]   s      r)   rO   zNemotronModel.__init__   s    	//!%!7!7"""LL$:J:JK16t7M7M1N
1NA$'1N
 ((8(8dmmL	
s   Cinputsc                     | j                  |      }|d gt        | j                        z  }t        ||d         }t	        | j                  |      D ]  \  }} ||||      } | j                  |      S )Nr   )r_   )r   lenr   r   zipr   )r(   r   r_   r   r^   layercs          r)   rF   zNemotronModel.__call__   su    
 f%=FS--E$Qa1DKK/HE1aQ'A 0 yy|r+   r7   r}   ru   s   @r)   r   r      s'    
MY 
M r+   r   c                   X     e Zd Zdef fdZ	 ddej                  fdZed        Z	 xZ
S )ModelrI   c                     t         |           || _        |j                  | _        t	        |      | _        |j                  s2t        j                  |j                  |j                  d      | _        y y )NFrK   )rN   rO   rI   r   r   modelr    r8   rS   r   r   lm_headr   s     r)   rO   zModel.__init__   sX    	//"4(
''99T%5%5tUSDL (r+   r   c                     | j                  ||      }| j                  j                  r'| j                   j                  j	                  |      }|S | j                  |      }|S r7   )r   rI   r    r   	as_linearr   )r(   r   r_   r   s       r)   rF   zModel.__call__   sW    
 jj'99((**))33C8C 
 ,,s#C
r+   c                 .    | j                   j                  S r7   )r   r   )r(   s    r)   r   zModel.layers   s    zz   r+   r7   )r,   r-   r.   r   rO   rB   rs   rF   propertyr   rt   ru   s   @r)   r   r      s;    TY T 

 ! !r+   r   )dataclassesr   	functoolsr   typingr   r   r   r   mlx.corecorerB   mlx.nnr8   rL   r
   r   r   r   compiler<   	LayerNormr>   ModulerH   rw   r   r   r   r4   r+   r)   <module>r      s    "  - -   T T  Y  Y  YF 	t$ %=",, =;#		 ;#|=")) =ryy 2BII <!BII !r+   