
    (HJj07                        d dl mZmZ d dlmZmZmZmZmZ d dl	m
Z d dlmZ ddlmZmZmZ ddlmZ ddlmZ  ed	       G d
 d             Z ed	       G d d             Z ed	       G d d             ZdededefdZdededefdZej:                  ej<                  ej>                  ej@                  ej@                  dZ!e G d de             Z" G d dejF                        Z$ G d dejF                        Z% G d dejF                        Z& G d  d!ejF                        Z' G d" d#ejF                        Z( G d$ d%ejF                        Z)y)&    )	dataclassfield)AnyDictListOptionalUnionN   )BaseModelArgscreate_attention_maskscaled_dot_product_attention)KVCache)initialize_ropeT)frozenc                       e Zd ZU dZeed<   dZeed<   dZee	e
      ed<   dZee   ed<   dZee   ed<   dZee   ed<   dZeed	<   dZeed
<   d Zy)AttentionConfigFno_opreplace_with_linearNsparsifyn_heads_in_groupwindow_lengthnum_sink_tokens$use_prefill_window_in_sink_attentionunshifted_sinkc                 V   | j                   s| j                  rFt        j                  | dd        t        j                  | dd        t        j                  | dd        y | j                   s?| j                  t        d      | j                  dk  rt        d| j                         y y )Nr   r   r   z>n_heads_in_group must be specified for active attention blocksr   z'n_heads_in_group must be positive, got )r   r   object__setattr__r   
ValueErrorselfs    d/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx_lm/models/nemotron-nas.py__post_init__zAttentionConfig.__post_init__   s    ::11t%7>t_d;t%6=$$, T  $$) =d>S>S=TU  *     )__name__
__module____qualname__r   bool__annotations__r   r   r   liststrr   intr   r   r   r   r"    r#   r!   r   r      su    E4 %%$(HhtCy!(&*hsm*#'M8C='%)OXc]) )$  !ND r#   r   c                   ^    e Zd ZU dZeed<   dZeed<   dZee	e
      ed<   dZee   ed<   d Zy)	FFNConfigFr   r   Nr   ffn_multc                    | j                   s| j                  rt        j                  | dd        y | j                   sC| j                  t        d      t        j                  | dt        | j                  d             y y )Nr/   z0ffn_mult must be specified for active FFN blocks   )r   r   r   r   r/   r   roundr   s    r!   r"   zFFNConfig.__post_init__5   sb    ::11tZ6}}$ !STTtZt}}a1HI	 r#   )r$   r%   r&   r   r'   r(   r   r   r   r)   r*   r/   floatr"   r,   r#   r!   r.   r.   .   s>    E4 %%$(HhtCy!( $Hhuo$	Jr#   r.   c                   8    e Zd ZU eed<   eed<   edefd       Zy)BlockConfig	attentionffndatac                     t        di |j                  di       }t        di |j                  di       } | ||      S )Nr6   r7   )r6   r7   r,   )r   getr.   )clsr8   	attn_confffn_confs       r!   	from_dictzBlockConfig.from_dictF   sA     $@dhh{B&?@	3txxr23YH55r#   N)	r$   r%   r&   r   r(   r.   classmethoddictr>   r,   r#   r!   r5   r5   A   s&    	N6T 6 6r#   r5   nkreturnc                 ,    | |z  dk(  r| S | |z   | |z  z
  S )z<Finds the smallest multiple of k greater than or equal to n.r   r,   )rA   rB   s     r!   _find_multiplerE   N   s$    1uzq5AE?r#   r/   n_embdc                 B    t        d| z  |z  dz        }t        |d      S )zQCalculates intermediate size based on multiplier, rounding up to multiple of 256.         )r+   rE   )r/   rF   intermediate_sizes      r!   _ffn_mult_to_intermediate_sizerL   U   s)    AL61A56+S11r#   )silurelugelugelu_new	gelu_fastc                       e Zd ZU dZeed<   dZeed<   dZeed<   dZ	eed<   d	Z
eed
<   dZeed<    ee      Zeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeeeeeef   f      ed<   dZeed<   dZeed<   d Zy)	ModelArgsznemotron-nas
model_typei    hidden_sizeP   num_hidden_layers@   num_attention_headsgh㈵>rms_norm_epsi  
vocab_size)default_factoryblock_configsrM   
hidden_actFattention_biasmlp_biasg    A
rope_thetaNrope_scalingi   max_position_embeddingstie_word_embeddingsc           
         | j                   rOt        | j                   d   t              r2| j                   D cg c]  }t        j	                  |       c}| _         t        | j                         | j                  k7  r/t        dt        | j                          d| j                   d      | j                  rAd| j                  vrt        d      | j                  j                  d      }|t        d      t        | j                         D ]p  \  }}|j                  }|j                  r|j                  r,| j                  |j                  z  dk7  sIt        d	| d
| j                   d|j                   d       y c c}w )Nr   zNumber of block_configs (z ) must match num_hidden_layers ()factorz"rope_scaling must contain 'factor'	rope_typez%rope_scaling must contain 'rope_type'zLayer z: num_attention_heads (z)) must be divisible by n_heads_in_group ()r]   
isinstancer@   r5   r>   lenrW   r   rb   r:   	enumerater6   r   r   rY   r   )r    confrh   i
block_confr<   s         r!   r"   zModelArgs.__post_init__v   s   *T-?-?-BD"I8<8J8J"8J%%d+8J"D t!!"d&<&<<+C0B0B,C+D E&&*&<&<%=Q@  t000 !EFF))--k:I  !HII 't'9'9:MAz",,I??9+H+H++i.H.HHAM$ #:4;S;S:T UBBKB\B\A]]^` 	 ;'"s   F)r$   r%   r&   rT   r*   r(   rU   r+   rW   rY   rZ   r3   r[   r   r)   r]   r^   r_   r'   r`   ra   rb   r   r   r	   rc   rd   r"   r,   r#   r!   rS   rS   e   s    $J$Ks!!L%J5M45J ND Hd J ;?L(4U5#:%6 678?#)S) %%r#   rS   c            	            e Zd ZdZdedef fdZ	 	 d
dej                  de	ej                     de	e
   dej                  fd	Z xZS )	Attentionz8Standard GQA Attention mechanism for layers that use it.argsattention_configc                    t         |           |j                  }|j                  x| _        }||j
                  z  x| _        }|j                  |z  x| _        }| j                  |z  |k7  rt        d| d| d      |dz  | _	        t        j                  |||z  |j                        | _        t        j                  |||z  |j                        | _        t        j                  |||z  |j                        | _        t        j                  ||z  ||j                        | _        t#        | j                  |j$                  d|j&                  |j(                        | _        y )Nzhidden_size (z,) must be divisible by num_attention_heads (rf   g      ࿩biasF)super__init__rU   rY   n_headsr   
n_kv_headshead_dimr   scalennLinearr_   q_projk_projv_projo_projr   ra   rb   rc   rope)r    rq   rr   dimrx   ry   rz   	__class__s          r!   rw   zAttention.__init__   sH   !%!9!99w'.2B2S2S'SS*#'#3#3w#>>MMG#+u$PQXPYYZ[  t^
iiWx%7d>Q>QRiiZ(%:ATATUiiZ(%:ATATUii( 2Cd>Q>QR $MMOO((
	r#   xmaskcacherC   c                 n   |j                   \  }}}| j                  |      | j                  |      | j                  |      }	}}|j	                  ||| j
                  | j                        j                  dddd      }|j	                  ||| j                  | j                        j                  dddd      }|	j	                  ||| j                  | j                        j                  dddd      }	|P| j                  ||j                        }| j                  ||j                        }|j                  ||	      \  }}	n"| j                  |      }| j                  |      }t        |||	|| j                  |      }
|
j                  dddd      j	                  ||d      }
| j                  |
      S )Nr   rH   r
   rI   )offset)r   r{   r   )shaper~   r   r   reshaperx   rz   	transposery   r   r   update_and_fetchr   r{   r   )r    r   r   r   BLDquerieskeysvaluesoutputs              r!   __call__zAttention.__call__   s    ''1a $AAAv//!QdmmDNNq!Q
 ||Aq$//4==AKKAqRSUVW1doot}}EOOq!Q
 iii=G99T%,,97D 11$?LD&ii(G99T?D-T6djjt
 !!!Q1-55aB?{{6""r#   NN)r$   r%   r&   __doc__rS   r   rw   mxarrayr   r   r   __classcell__r   s   @r!   rp   rp      sb    B
Y 
/ 
@ $(#	#88# rxx # }	#
 
#r#   rp   c                   L     e Zd ZdZdedef fdZdej                  fdZ	 xZ
S )MLPz5Standard Feed-Forward Network for layers that use it.rq   
ffn_configc                    t         |           |j                  }t        |j                  |      }t        j                  |||j                        | _        t        j                  |||j                        | _	        t        j                  |||j                        | _
        |j                  | _        | j                  t        vrt        d|j                         y )Nrt   zUnknown activation function: )rv   rw   rU   rL   r/   r|   r}   r`   	gate_proj	down_projup_projr^   act_fn_ACT2FNr   )r    rq   r   r   
hidden_dimr   s        r!   rw   zMLP.__init__   s    3J4G4GM
3
G:sGyyjt}}Eoo;;g%<T__<MNOO &r#   rC   c                     t         | j                     }| j                   || j                  |            | j	                  |      z        S N)r   r   r   r   r   )r    r   r   s      r!   r   zMLP.__call__   s:    %~~fT^^A%67$,,q/IJJr#   )r$   r%   r&   r   rS   r.   rw   r   r   r   r   r   s   @r!   r   r      s.    ?PY PI PKRXX Kr#   r   c                   d     e Zd ZdZdedef fdZdej                  dej                  fdZ	 xZ
S )LinearSubblockReplacementz>A simple linear layer used to replace Attention or MLP blocks.rU   ru   c                 \    t         |           t        j                  |||      | _        y )Nrt   )rv   rw   r|   r}   linear)r    rU   ru   r   s      r!   rw   z"LinearSubblockReplacement.__init__   s"    ii[tDr#   r   rC   c                 $    | j                  |      S r   )r   )r    r   rq   kwargss       r!   r   z"LinearSubblockReplacement.__call__   s    {{1~r#   )r$   r%   r&   r   r+   r'   rw   r   r   r   r   r   s   @r!   r   r      s7    HEC Et E"((  r#   r   c            	            e Zd ZdZdedef fdZ	 	 d
dej                  de	ej                     de	e
   dej                  fd	Z xZS )TransformerBlockzFA single transformer block, potentially heterogeneous based on config.rq   	layer_idxc                    t         |           |j                  | _        |j                  |   }|j                  | _        |j                  | _        | j
                  j                  s1t        j                  |j                  |j                        | _        nd | _        | j
                  j                  rd | _        nW| j
                  j                  r&t        |j                  |j                         | _        nt#        || j
                        | _        | j                  j                  s1t        j                  |j                  |j                        | _        nd | _        | j                  j                  rd | _        y | j                  j                  r&t        |j                  |j(                        | _        y t+        || j                        | _        y )Neps)rv   rw   rU   r]   r6   rr   r7   r   r   r|   RMSNormrZ   input_layernorm	self_attnr   r   r_   rp   post_attention_layernormmlpr`   r   )r    rq   r   block_configr   s       r!   rw   zTransformerBlock.__init__   sc   ++)))4 , 6 6&** $$**#%::d.>.>DDUDU#VD #'D    &&!DN""666  $"5"5DN
 'tT-B-BCDN $$,.JJ  d&7&7-D) -1D) ??  DH__0001A1A4==QDH 41DHr#   r   r   r   rC   c                     | j                   ,|}| j                  |      }| j                  |||      }||z   }| j                  )|}| j                  |      }| j                  |      }||z   }|S )N)r   r   )r   r   r   r   )r    r   r   r   residualhattn_outmlp_outs           r!   r   zTransformerBlock.__call__'  s~     >>%H$$Q'A~~ad%~@H8#A 88H--a0AhhqkG7"Ar#   r   )r$   r%   r&   r   rS   r+   rw   r   r   r   r   r   r   r   s   @r!   r   r      sb    P(2Y (23 (2Z $(#	88 rxx  }	
 
r#   r   c                   \     e Zd ZdZdef fdZ	 ddej                  dee	e
      fdZ xZS )NemotronNASModelz.The core Nemotron-NAS style transformer model.rq   c                    t         |           || _        |j                  | _        |j                  | _        t        j                  |j                  |j                        | _        t        |j                        D cg c]  }t        ||       c}| _        t        j                  |j                  |j                        | _        t        d | j                  D              | _        y c c}w )N)rq   r   r   c              3   :   K   | ]  }|j                   d  y w)Nr
   )r   ).0layers     r!   	<genexpr>z,NemotronNASModel.__init__.<locals>.<genexpr>M  s      #
&%%//*EA;s   )rv   rw   rq   r[   rW   r|   	EmbeddingrU   embed_tokensranger   layersr   rZ   normsumnum_attn_layers)r    rq   rm   r   s      r!   rw   zNemotronNASModel.__init__B  s    	//!%!7!7LL$:J:JK 4112
2 $!42
 JJt//T5F5FG	" #
;;#
  

s   ?C-inputsr   c                     | j                  |      }|d g| j                  z  }t        ||d         }d}| j                  D ]&  }|j                  ||   }|dz  }nd } ||||      }( | j                  |      S )Nr   r
   r   )r   r   r   r   r   r   )r    r   r   r   r   	cache_idxr   cs           r!   r   zNemotronNASModel.__call__Q  s    
 f%=FT111E$Qa1	[[E*)$Q	aQ'A ! yy|r#   r   )r$   r%   r&   r   rS   rw   r   r   r   r   r   r   r   r   s   @r!   r   r   ?  s;    8
Y 
$ &* S	"r#   r   c                   d     e Zd Zdef fdZ	 ddej                  fdZd Ze	d        Z
d Z xZS )	Modelrq   c                     t         |           || _        |j                  | _        t	        |      | _        |j                  s2t        j                  |j                  |j                  d      | _        y d | _        y )NFrt   )rv   rw   rq   rT   r   modelrd   r|   r}   rU   r[   lm_head)r    rq   r   s     r!   rw   zModel.__init__k  sZ    	//%d+
''99T%5%5tUSDLDLr#   r   c                     | j                  ||      }| j                  j                  r'| j                   j                  j	                  |      }|S | j                  |      }|S )Nr   )r   rq   rd   r   	as_linearr   )r    r   r   outs       r!   r   zModel.__call__u  sY    
 jjuj-99((**))33C8C 
 ,,s#C
r#   c                 V    | j                   j                  r|j                  dd        |S )Nzlm_head.weight)rq   rd   pop)r    weightss     r!   sanitizezModel.sanitize  s#    99((KK($/r#   c                 .    | j                   j                  S r   )r   r   r   s    r!   r   zModel.layers  s    zz   r#   c                 j    | j                   D cg c]  }|j                  t                c}S c c}w r   )r   r   r   )r    r   s     r!   
make_cachezModel.make_cache  s)    '+{{R{eeoo6Q	{RRRs   00r   )r$   r%   r&   rS   rw   r   r   r   r   propertyr   r   r   r   s   @r!   r   r   i  sD     Y   


 ! !Sr#   r   )*dataclassesr   r   typingr   r   r   r   r	   mlx.corecorer   mlx.nnr|   baser   r   r   r   r   
rope_utilsr   r   r.   r5   r+   rE   r3   rL   rM   rN   rO   gelu_approxr   rS   Modulerp   r   r   r   r   r   r,   r#   r!   <module>r      sd   ) 3 3   T T  ' $  > $J J J$ $	6 	6 	6c c c 2U 2C 2C 2 GGGGGG - - -`>#		 >#BK")) K.			 	Bryy BJ'ryy 'T"SBII "Sr#   