
    (HJj
3                        d dl Z d dlmZ d dlmZmZ d dlmZ d dl	m
Z
 ddlmZmZmZ ddlmZmZ e G d de             Z G d	 d
e
j&                        Zd Z G d de
j&                        Z G d de
j&                        Z G d de
j&                        Z G d de
j&                        Z G d de
j&                        Z G d de
j&                        Z G d de
j&                        Z G d de
j&                        Zy)    N)	dataclass)ListOptional   )BaseModelArgscreate_attention_maskscaled_dot_product_attention)ArraysCacheRotatingKVCachec                       e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed	<   eed
<   eed<   eed<   eed<   dZeed<   dZ	e
ee      ed<   dZe
ee      ed<   d Zy)	ModelArgs
model_typeattention_biasconv1d_widthhidden_sizeintermediate_sizelogits_soft_capnum_attention_headsnum_hidden_layersnum_key_value_headsrms_norm_eps
rope_thetaattention_window_size
vocab_sizeTembeddings_scale_by_sqrt_dimNblock_types_block_typesc                 @    | j                   | j                  | _         y y N)r   r   selfs    g/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx_lm/models/recurrent_gemma.py__post_init__zModelArgs.__post_init__!   s!    ##00D $    )__name__
__module____qualname__str__annotations__boolintfloatr   r   r   r   r   r#    r$   r"   r   r      s    OO)- $-'+K$s)$+(,L(49%,1r$   r   c                   0     e Zd Zddedef fdZd Z xZS )RMSNormdimsepsc                 f    t         |           t        j                  |f      | _        || _        y r   )super__init__mxonesweightr1   )r!   r0   r1   	__class__s      r"   r4   zRMSNorm.__init__(   s'    ggtg&r$   c                 r    t         j                  j                  |d| j                  z   | j                        S )N      ?)r5   fastrms_normr7   r1   )r!   xs     r"   __call__zRMSNorm.__call__-   s(    ww3#4dhh??r$   )gh㈵>)r%   r&   r'   r+   r,   r4   r>   __classcell__r8   s   @r"   r/   r/   '   s    S u 
@r$   r/   c                 8   | j                   dk(  sJ |j                  | j                  |j                    d  k(  sJ |j                  | j                  k(  sJ | j                  d   dk(  r'|| | d d df   fS ||d d d f   z  | z   }||d d df   fS ||}n3| j                  \  }}}t        j                  ||f| j                        }t        j
                  |       }t        | j                  d         D ]!  }|d d |f   |z  | d d |f   z   }||d d |f<   # ||fS )N   r   r   )dtype)ndimshaperD   r5   zeros
zeros_likerange)	r=   ah0yh_tB_Dts	            r"   rnn_scanrR   1   s)   66Q;;77aggqvvgi((((77aggwwqzQ:a1g: Bq$wK!#Aa2h; >CggGAq!((Aq61CMM!qwwqz"AAqD'C-!AqD')CAadG # c6Mr$   c                   0     e Zd Zdedef fdZddZ xZS )Conv1dchannelskernel_sizec                     t         |           t        j                  ||df      | _        t        j                  |f      | _        y )Nr   )r3   r4   r5   rG   r7   bias)r!   rU   rV   r8   s      r"   r4   zConv1d.__init__P   s9    
 	hh+q9:HHh[)	r$   c                 T   |j                   \  }}}| j                  j                   \  }}}|t        j                  ||gd      }nt        j                  |d|dz
  dfdg      }t        j
                  || j                  |      }	|	| j                  z   }	|	|d d | dz   d d d f   fS )Nr   )axis)r   r   r   )groups)rF   r7   r5   concatenatepadconv_generalrX   )
r!   r=   cacherN   LCr[   KrO   rL   s
             r"   r>   zConv1d.__call__Y   s    ''1a{{((1qz2Aq6AE1:v67AOOAt{{6:		M!ArAvxN###r$   r   )r%   r&   r'   r+   r4   r>   r?   r@   s   @r"   rT   rT   O   s    ** *$r$   rT   c                   P     e Zd ZdZdedef fdZ	 ddej                  fdZ xZ	S )RGLRUz2A Real-Gated Linear Recurrent Unit (RG-LRU) layer.width	num_headsc                 p   t         |           || _        || _        | j                  | j                  z  | _        t        j                  | j                  f      | _        t        j                  | j                  | j                  | j                  f      | _        t        j                  | j                  | j                  f      | _	        t        j                  | j                  | j                  | j                  f      | _
        t        j                  | j                  | j                  f      | _        y r   )r3   r4   re   rf   head_dimr5   rG   recurrent_paraminput_gate_weightinput_gate_biasrecurrent_gate_weightrecurrent_gate_bias)r!   re   rf   r8   s      r"   r4   zRGLRU.__init__k   s    
 	
"

dnn4!xx6!#^^T]]DMM:"
  "xx(GH%'XX^^T]]DMM:&
" $&88T^^T]],K#L r$   r=   c                     |j                   \  } fd} || j                   j                        } || j                   j                        }d|z  t        j                   j                        z  }t        j                  |      }t        j                  d|z        }	||z  }
t        j                  d|	z
        }|d|d d dd d f<   |
|j                  |j                        z  }t        |||      \  }}||fS )Nc                     | j                  j                  j                  f      } | j                  dd      |z  j                  dd      |z   } t	        j
                  | j                  dd            S )Nr      rB   )reshaperf   rh   swapaxesr5   sigmoidflatten)hwbrN   r`   r!   s      r"   apply_block_linearz*RGLRU.__call__.<locals>.apply_block_linear   sb    		1a?@AAq!A%//159A::aii1o..r$   g       rp   r   r:   r   )r=   rJ   rK   )rF   rj   rk   rl   rm   nnsoftplusri   r5   expsqrtastyperD   rR   )r!   r=   r_   rO   rx   gate_xgate_alog_arJ   a_squaregated_x
multipliernormalized_xrL   last_hrN   r`   s   `              @@r"   r>   zRGLRU.__call__   s    
 ''1a	/ $At'='=t?S?ST#t))4+C+C

 vD,@,@ AAFF5M66!e)$ f* WWQ\*
="%Jq!Qw!2!2177!;;
	6 &yr$   r   )
r%   r&   r'   __doc__r+   r4   r5   arrayr>   r?   r@   s   @r"   rd   rd   h   s6    <MM M2 &88&r$   rd   c            	       \     e Zd Z	 	 ddedededef fdZ	 	 d	dej                  fdZ xZS )
RecurrentBlockre   rf   	lru_widthconv1d_temporal_widthc                    t         |           || _        || _        |xs || _        || _        t        j                  || j                        | _        t        j                  || j                        | _	        t        j                  | j                  |      | _
        t        | j                  | j
                        | _        t        | j                  | j                        | _        y )N)rU   rV   )re   rf   )r3   r4   re   rf   r   r   ry   Linearlinear_ylinear_x
linear_outrT   conv_1drd   rg_lru)r!   re   rf   r   r   r8   s        r"   r4   zRecurrentBlock.__init__   s     	
""+e%:"		%8		%8))DNNE:^^22
 ..nn
r$   r=   c                    | j                  |      }t        j                  |      }| j                  |      }|d d g}| j	                  ||d         \  }|d<   | j                  ||d         \  }|d<   ||z  }| j                  |      }|S )Nr   )r=   r_   r   )r   ry   gelu_approxr   r   r   r   )r!   r=   r_   maskrL   s        r"   r>   zRecurrentBlock.__call__   s     MM!NN1 MM!=4LEllQeAhl758kkAU1Xk658EOOAr$   N   NN	r%   r&   r'   r+   r4   r5   r   r>   r?   r@   s   @r"   r   r      sO     %&

 
 	

  #
8 	88r$   r   c                   R     e Zd Zdededef fdZ	 	 ddej                  fdZ xZS )LocalAttentionBlockre   rf   window_sizec                 |   t         |           || _        || _        || _        ||z  dz  | _        | j                  | j                  z  | _        t        j                  | j                  | j                  d      | _	        t        j                  | j                  | j                  d      | _
        t        j                  | j                  | j                  d      | _        t        j                  | j                  | j                  d      | _        t        j                  | j                  dz  d      | _        y )Ng      FrX   Trp   )traditional)r3   r4   re   rf   r   scalerh   ry   r   q_projk_projv_projo_projRoPErope)r!   re   rf   r   r8   s       r"   r4   zLocalAttentionBlock.__init__   s     	
"&y(d3


dnn4ii

DJJUCii

DMMFii

DMMFii

DJJTBGGMMQ
	r$   r=   c                 
   |j                   \  }}}| j                  |      | j                  |      | j                  |      }	}}|j	                  ||| j
                  d      j                  dddd      }|j	                  ||dd      j                  dddd      }|	j	                  ||dd      j                  dddd      }	|P| j                  ||j                        }| j                  ||j                        }|j                  ||	      \  }}	n"| j                  |      }| j                  |      }t        |||	|| j                  |      }
|
j                  dddd      j	                  ||d      }
| j                  |
      S )NrC   r   rp   r   rB   )offset)r_   r   r   )rF   r   r   r   rq   rf   	transposer   r   update_and_fetchr	   r   r   )r!   r=   r_   r   rN   r`   rP   querieskeysvaluesoutputs              r"   r>   zLocalAttentionBlock.__call__   s_    ''1a $AAAv//!Q;EEaAqQ||Aq!R(221aA>1a,66q!QBiii=G99T%,,97D 11$?LD&ii(G99T?D-T6djjt
 !!!Q1-55aB?{{6""r$   r   r   r@   s   @r"   r   r      s=    

 
 	
2 	#88#r$   r   c                   H     e Zd Zdedef fdZdej                  fdZ xZS )MLPBlockre   expanded_widthc                     t         |           t        j                  ||dz        | _        t        j                  ||dz        | _        t        j                  |dz  |      | _        y )Nrp   )r3   r4   ry   r   up_proj	gate_proj	down_proj)r!   re   r   r8   s      r"   r4   zMLPBlock.__init__  sS    yy!(;<5.A*=>>Q#6>r$   r=   c                     | j                  |      }| j                  |      }| j                  t        j                  |      |z        S r   )r   r   r   ry   r   )r!   r=   gates      r"   r>   zMLPBlock.__call__  s:    ~~a LLO~~bnnT2Q677r$   r   r@   s   @r"   r   r     s&    ?c ?3 ?8"(( 8r$   r   c                   n     e Zd Z	 	 ddedededededee   def fdZ	 	 dd	ej                  fd
Z	 xZ
S )ResidualBlockre   mlp_expanded_widthrf   r   temporal_block_typer   r   c                 
   t         |           || _        || _        || _        || _        || _        || _        || _        t        |      | _
        | j                  dk(  r=t        | j                  | j                  | j                  | j                        | _        n1t        | j                  | j                  | j
                        | _        t        |      | _        t        | j                  | j                        | _        y)aV  Initializes the residual block.

        Args:
          width: The width of the block.
          mlp_expanded_width: The width of the expansion inside the MLP block.
          num_heads: The number of heads for the Attention or the RG-LRU.
          attention_window_size: The window size for the local attention block.
          temporal_block_type: Either "recurrent" or "attention", specifying the
            type of recurrent block to use.
          lru_width: The width of the RG-LRU if different from `width`.
          conv1d_temporal_width: The width of the temporal convolution.
        	recurrent)re   rf   r   r   )re   rf   r   )re   r   N)r3   r4   re   r   rf   r   r   r   r   r/   temporal_pre_normr   temporal_blockr   channel_pre_normr   	mlp_block)	r!   re   r   rf   r   r   r   r   r8   s	           r"   r4   zResidualBlock.__init__!  s    , 	
"4"%:"#6 "%:"!(##{2"0jj....&*&@&@	#D #6jj.. 66#D !(!**22
r$   r=   c                     |}| j                  |      }| j                  |||      }||z   }| j                  |      }| j                  |      }||z   }|S )N)r_   r   )r   r   r   r   )r!   r=   r_   r   raw_xinputs_normalizedresiduals          r"   r>   zResidualBlock.__call__V  sg      2259 1TJu9!!(+NN1Lr$   r   r   )r%   r&   r'   r+   r(   r   r4   r5   r   r>   r?   r@   s   @r"   r   r     sr     $(%&3
3
  3
 	3

  #3
 !3
 C=3
  #3
p 	88r$   r   c                   (     e Zd Z fdZ	 ddZ xZS )Griffinc                 H   t         |           || _        t        j                  |j
                  |j                        | _        |j                  | _	        |j                  }t        |j                        D cg c]J  }t        |j                  |j                  |j                  |j                   ||t#        |      z     d       L c}| _        t'        |j                  |j(                        | _        |j                   | _        |j/                  d      | _        y c c}w )N)re   r   rf   r   r   r   )r1   	attention)r3   r4   configry   	Embeddingr   r   embed_tokensr   scale_by_sqrt_dimr   rI   r   r   r   r   r   lenlayersr/   r   
final_normr   indexswa_idx)r!   r   r   ir8   s       r"   r4   zGriffin.__init__l  s   LL

 "(!D!D(( 6334

 5 ((#)#;#; 44&,&B&B$/C4D0D$E 5

 "&"4"4&:M:MN!77"((5

s   :ADc                 ~   | j                  |      }| j                  r%|t        j                  |j                  d         z  }|d gt        | j                        z  }t        ||| j                     | j                        }t        | j                        D ]  \  }} |||||         } | j                  |      S )NrC   )r   )r   r_   )r   r   mathr|   rF   r   r   r   r   r   	enumerater   )r!   tokensr_   r=   r   r   blocks          r"   r>   zGriffin.__call__  s    
 f%!!DIIaggbk**A=FS--E$uT\\"0@0@
 "$++.HAuad%(3A / q!!r$   r   )r%   r&   r'   r4   r>   r?   r@   s   @r"   r   r   k  s    6< "r$   r   c                   h    e Zd Zd Zd	dej
                  dej
                  fdZed        Zd Z	d Z
y)
Modelc                     || _         t        |      | _        |j                  | _        t	        j
                  |j                  |j                  d      | _        y )NFr   )	argsr   modelr   ry   r   r   r   lm_head)r!   r   s     r"   r4   zModel.__init__  sA    	V_
 ++yy!3!3V5F5FUSr$   Nr   returnc                    | j                  ||      }d| v r| j                  |      }n%| j                   j                  j                  |      }| j                  j
                  }|rt        j                  ||z        |z  }|S )N)r_   r   )r   r   r   	as_linearr   r   r5   tanh)r!   r   r_   logitscs        r"   r>   zModel.__call__  sr    F%0\\&)FZZ,,66v>FII%%WWVaZ(1,Fr$   c                 .    | j                   j                  S r   )r   r   r    s    r"   r   zModel.layers  s    zz   r$   c                     |j                         D ]2  \  }}d|v s|j                  d   dk7  s|j                  dd      ||<   4 d|vr| j                  d       |S )Nzconv_1d.weightrC   r   rp   zlm_head.weightr   )itemsrF   moveaxispop)r!   weightskvs       r"   sanitizezModel.sanitize  s\    MMODAq1$)9ZZ1-
 $ 7*HHYr$   c                     g }| j                   D ]\  }|j                  dk(  r|j                  t        d             .|j                  t	        | j
                  j                               ^ |S )Nr   rp   )size)max_size)r   r   appendr
   r   r   r   )r!   r_   layers      r"   
make_cachezModel.make_cache  sV    [[E((K7[a01_dii6U6UVW	 !
 r$   r   )r%   r&   r'   r4   r5   r   r>   propertyr   r   r   r-   r$   r"   r   r     sA    T
rxx 
 
 ! !r$   r   )r   dataclassesr   typingr   r   mlx.corecorer5   mlx.nnry   baser   r   r	   r_   r
   r   r   Moduler/   rR   rT   rd   r   r   r   r   r   r   r-   r$   r"   <module>r      s     ! !   T T / 1 1 10@bii @<$RYY $2?BII ?D/RYY /d2#")) 2#j8ryy 8IBII IX/"bii /"d'BII 'r$   