
    (HJj:                         d dl Z d dlmZ d dlmZmZ d dlmZ d dl	m
Z
 d dlmZmZmZ ddlmZ ddlmZmZ ddlmZ e G d	 d
e             Z G d de
j.                        Z G d de
j.                        Z G d de
j.                        Z G d de
j.                        Z G d de
j.                        ZdededefdZ G d de
j.                        Z  G d de
j.                        Z! G d de
j.                        Z"y)    N)	dataclass)AnyOptional)BaseModelArgscreate_attention_maskcreate_ssm_mask   )swiglu)ArraysCacheKVCache)
ssm_updatec                   $   e Zd ZU dZeed<   dZeed<   dZeed<   dZ	e
ed<   d	Zeed
<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeee      ed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   d	Zeed<   dZeed<   d Zeed!<   y)"	ModelArgsplamo2
model_typei   hidden_size    num_hidden_layersư>rms_norm_epsTtie_word_embeddingsnum_attention_heads   num_key_value_heads   hidden_size_per_headi   max_position_embeddingsattention_window_sizeNfull_attention_idx@   mamba_d_statemamba_d_convmamba_num_heads   
mamba_step   mamba_chunk_sizemamba_enabledi 4  intermediate_sizei }  
vocab_size)__name__
__module____qualname__r   str__annotations__r   intr   r   floatr   boolr   r   r   r   r   r   r   listr!   r"   r#   r%   r'   r(   r)   r*        ^/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx_lm/models/plamo2.pyr   r      s    JKsL% $$!!   ####'S'!%3%.2c+2M3L#OSJcM4"s"Jr5   r   c            	       n     e Zd Z	 	 d	dedededdf fdZdej                  dej                  fdZ xZ	S )
RMSNormr   epsoffsetreturnNc                 r    t         |           t        j                  |      | _        || _        || _        y N)super__init__mxzerosweightvariance_epsilonr:   )selfr   r9   r:   	__class__s       r6   r?   zRMSNorm.__init__)   s/     	hh{+ #r5   hidden_statesc                     t         j                  j                  || j                  | j                  z   | j
                        S r=   )r@   fastrms_normrB   r:   rC   )rD   rF   s     r6   __call__zRMSNorm.__call__4   s3    ww4;;4d6K6K
 	
r5   )r         ?)
r+   r,   r-   r0   r1   r?   r@   arrayrJ   __classcell__rE   s   @r6   r8   r8   (   sQ     			 	 		
 
	
bhh 
288 
r5   r8   c                   t    e Zd Zdeddf fdZdej                  dee   deej                     dej                  fdZ	d	ej                  d
ej                  dej                  dej                  dee
   deej                     dej                  fdZ	 	 ddej                  deej                     fdZ xZS )Mambaconfigr;   Nc                    t         |           || _        |j                  | _        |j                  | _        |j                  | _        |j                  | _	        |j                  | _        |j                  | _        | j                  | j                  z  | _        t        j                  | j                  d| j                  z  d      | _        t        j"                  | j                  | j                  d| j                  | j                  d      | _        t'        d| j                  dz        | _        t        j                  | j                  | j(                  d| j
                  z  z   d      | _        t        j                  | j(                  | j                  d      | _        t/        j0                  | j                  f      | _        t/        j4                  t/        j6                  d	| j                  d	z   t.        j8                  
            | _        t/        j<                  | j                        | _        t/        j<                  | j(                        | _         t/        j<                  | j
                        | _!        t/        j<                  | j
                        | _"        t        j                  | j                  | j                  d      | _#        y )Nr$   Fbiasr   )in_channelsout_channelsrT   kernel_sizegroupspaddingr       )shaper	   dtype)$r>   r?   rQ   r   r!   d_stater"   conv_kernel_sizer'   
chunk_sizer#   	num_headsr   r)   nnLinearin_projConv1dconv1dmaxdt_dim	bcdt_projdt_projr@   rA   dt_biaslogarangefloat32A_logonesDdt_norm_weightB_norm_weightC_norm_weightout_projrD   rQ   rE   s     r6   r?   zMamba.__init__;   s    !--++ & 3 3 11//$*$?$?!!%$2K2K!Kyya$"8"88u
 ii..//--))
 "d.."45""KK!dll**

 yydnn5Ixxt~~&78VVBIIa!);2::NO
( ggdkk2WWT\\2WWT\\2		$"8"8$:J:JQVWr5   
conv_inputcachemaskc                    |t        j                  |d   |d      }||d   It        j                  |j                  d   | j                  dz
  | j
                  f|j                        }n|d   }t        j                  ||gd      }| j                  dz
  }|j                  q|j                  d   }t        j                  |j                  d||z
        }|d d d f   t        j                  |      z   d   }	t        j                  ||	d      |d<   n;|d d | d d d f   |d<   n(t        j                  |d| j                  dz
  dfdg      }| j                  |      }
t        j                  |
      S )N).Nr   r	   r\   axis)r   r   )r@   whererA   r[   r_   r)   r]   concatenatelengthscliprm   take_along_axispadrf   rb   silu)rD   rw   rx   ry   
conv_statepadded_inputn_keeptends	positionsconv_outputs              r6   _convzMamba._conve   sp    $y/:qAJQxXX"((+--1..
 %**
 #1X
>>:z*BKL**Q.F}}( &&q)wwu}}aV<!!T']RYYv->>	J	--lIANa'F78Q7a66Vd&;&;a&?%CVLL kk,/ww{##r5   xBCdtc                    |j                   \  }}}	|j                  ||| j                  | j                        }|j                  ||d| j                        }|j                  ||d| j                        }|r|d   }
|j
                  }nd\  }
}t        || j                  ||| j                  || j                  |
||
      \  }}
|r|
|d<   |j                  ||| j                        S )Nr	   NN)ry   r   )r[   reshapera   r   r^   r   r   ro   rq   rk   r)   )rD   r   r   r   r   rx   ry   
batch_sizeseq_len_stater   ys                r6   _ssmz
Mamba._ssm   s     "#
GQIIj'4>>4;T;TUIIj'1dll;IIj'1dll;!HEmmG'NE7JJFFLL
5 E!HyyWd.D.DEEr5   rF   c                    |j                   \  }}}| j                  |      }|j                  ||| j                  d      }t	        j
                  || j                  gd      \  }}	|	j                  |d| j                  | j                  z        }	| j                  |	||      }	| j                  |	      }
t	        j
                  |
| j                  | j                  dz  gd      \  }}}t	        j                  | j                  j                  t        j                               }t        j                  j                  || j                   | j"                  j$                        }t        j                  j                  || j&                  | j"                  j$                        }t        j                  j                  || j(                  | j"                  j$                        }| j+                  |      }| j-                  |	|||||      }|r|j/                  |j                   d          t1        |j3                  d      |      }| j5                  |      S )Nr{   r$   r	   )r[   rd   r   ra   r@   splitr   r   ri   r^   expro   astypern   rH   rI   rr   rQ   r   rs   rt   rj   r   advancer
   flattenru   )rD   rF   ry   rx   bsizelengthr   zxzr   BCdtr   r   r   Aouts                   r6   rJ   zMamba.__call__   s    )..vq\\-(ZZvt~~r: xx)) 
1 IIeR$2K2K!KLJJq%&~~a 88D4<<1A"BL1bVVDJJ%%bjj122WWb$"5"5t{{7O7OPGGQ 2 2DKK4L4LMGGQ 2 2DKK4L4LM \\"ii
 MM#))A,'QYYr]C(}}S!!r5   r   )r+   r,   r-   r   r?   r@   rL   r   r   r   r   r   rJ   rM   rN   s   @r6   rP   rP   :   s    (Xy (XT (XT$$HH$$ $$$ rxx 	$$
 
$$L"F88"F 88"F 88	"F
 HH"F }"F rxx "F 
"FN $(	-"xx-" rxx -"r5   rP   c                   l     e Zd Zdeddf fdZ	 	 ddej                  deej                     fdZ xZ	S )		AttentionrQ   r;   Nc                 T   t         |           || _        |j                  | _        |j                  }|j
                  | _        |dz  | _        |j                  | _        |x| _	        | _
        |j                  x| _        | _        | j                  | j                  z  dk(  sJ | j                  | j                  z  | _        | j                  | j                  z  | _        | j                  | j                  z  | _        | j                  | j                  z  | _        t%        j&                  | j                  | j                  | j                   z   | j"                  z   d      | _        t%        j&                  | j                  | j                  z  | j                  d      | _        t-        j.                  | j                  | j                  f      | _        t-        j.                  | j                  | j                  f      | _        t%        j4                  | j                        | _        y )Ng      r   FrS   )r>   r?   rQ   r   r   r   scaler   q_num_headsqk_dimv_dimr   k_num_headsv_num_headsn_group
q_proj_dim
k_proj_dim
v_proj_dimrb   rc   qkv_projo_projr@   rp   q_weightk_weightRoPErope)rD   rQ   head_dimrE   s      r6   r?   zAttention.__init__   s   !--..'-'E'E$t^
!55#++dj.4.H.HH4+$"2"22a777''4+;+;;**T[[8**T[[8**TZZ7		OOdoo-?

 iitzz)4+;+;%
 !1!14;; ?@!1!14;; ?@GGDKK(	r5   rF   ry   c                    |j                   \  }}}| j                  |      }t        j                  || j                  | j                  | j
                  z   gd      \  }}	}
|j                  ||| j                  | j                        j                  dddd      }|	j                  ||| j                  | j                        j                  dddd      }	|
j                  ||| j                  | j                        j                  dddd      }
t        j                  j                  |d d      | j                  d d d f   z  }t        j                  j                  |	d d      | j                   d d d f   z  }	|P| j#                  ||j$                  	      }| j#                  |	|j$                  	      }	|j'                  |	|
      \  }	}
n"| j#                  |      }| j#                  |	      }	t        j                  j)                  ||	|
| j*                  |
      }|j                  dddd      j                  ||| j                  | j                  z        }| j-                  |      S )Nr   r{   r   r$   r	      r   )rB   r9   )r:   )r   ry   )r[   r   r@   r   r   r   r   r   r   	transposer   r   r   rH   rI   r   r   r   r:   update_and_fetchscaled_dot_product_attentionr   r   )rD   rF   ry   rx   r   Tr   qkvqkvoutputs               r6   rJ   zAttention.__call__   s     %%1ammM*(($//4??T__#DEB
1a IIaD,,dkk:DDQ1aPIIaD,,dkk:DDQ1aPIIaD,,djj9CCAq!QOGGQt6q$w9OOGGQt6q$w9OO		!ELL	1A		!ELL	1A))!Q/DAq		!A		!A55** 6 
 !!!Q1-55q$""TZZ/
 {{6""r5   r   )
r+   r,   r-   r   r?   r@   rL   r   rJ   rM   rN   s   @r6   r   r      sC    )y )T )D $(	%#xx%# rxx %#r5   r   c                   `     e Zd Zdeddf fdZdej                  dej                  fdZ xZS )MLPrQ   r;   Nc                 >   t         |           || _        |j                  | _        |j                  | _        t        j                  | j                  | j                  dz  d      | _        t        j                  | j                  | j                  d      | _        y )Nr$   FrS   )	r>   r?   rQ   r   r)   rb   rc   gate_up_proj	down_projrv   s     r6   r?   zMLP.__init__(  s}    !--!'!9!9IId44q8u
 4#9#94;K;KRWXr5   r   c                     | j                  |      }t        j                  |dd      }| j                  t	        |d   |d               S )Nr$   r   r{   r   r	   )r   r@   r   r   r
   )rD   r   hhss       r6   rJ   zMLP.__call__2  sB    a XXa$~~fRUBqE233r5   	r+   r,   r-   r   r?   r@   rL   rJ   rM   rN   s   @r6   r   r   '  s4    Yy YT Y4"(( 4rxx 4r5   r   c                   p     e Zd Zdededdf fdZ	 	 d	dej                  deej                     fdZ	 xZ
S )
PlamoDecoderLayerrQ   is_mambar;   Nc                    t         |           || _        |j                  | _        || _        |  |rt        |      | _        nt        |      | _        t        |      | _	        t        |j                  |j                  d      | _        t        |j                  |j                  d      | _        t        |j                  |j                  d      | _        t        |j                  |j                  d      | _        y )NrK   )r9   r:   g?gWfѷ?)r>   r?   rQ   r   r   rP   mixerr   r   mlpr8   r   pre_mixer_normpost_mixer_normpre_mlp_normpost_mlp_norm)rD   rQ   r   rE   s      r6   r?   zPlamoDecoderLayer.__init__9  s    !-- vDJ"6*DJv;%F$7$7
  'F$7$7 
 $F$7$7
 %F$7$7
r5   rF   ry   c                     |}| j                  |      }| j                  |||      }| j                  |      }||z   }|}| j                  |      }| j	                  |      }| j                  |      }||z   S )N)rF   ry   rx   )r   r   r   r   r   r   )rD   rF   ry   rx   residualhidden_states_sahidden_states_mlps          r6   rJ   zPlamoDecoderLayer.__call__Q  s     !++M:::' & 
  //0@A #33 ))-8 !HH]3 !../@A+++r5   r   )r+   r,   r-   r   r2   r?   r@   rL   r   rJ   rM   rN   s   @r6   r   r   8  sI    
y 
D 
T 
6 $(	,xx, rxx ,r5   r   rQ   ir;   c                     | j                   sy| j                  dkD  sJ || j                  k  sJ | j                  | j                  dz  k  r|| j                  dz
  k7  S || j                  z  | j                  dz  k7  S )NFr	   r$   )r(   r%   r   )rQ   r   s     r6   r   r   n  s    q   v'''''F$5$5$:;F,,q000!!!v'8'8A'=>>r5   c                   H     e Zd Zdeddf fdZdej                  fdZ xZS )PlamoDecoderrQ   r;   Nc           
          t         |           t        |j                        D cg c]  }t	        |t        ||             c}| _        |j                  rdnd | _        |j                  dz  | _
        y c c}w )N)r   r   r$   )r>   r?   ranger   r   r   layersr(   ssm_idxr%   fa_idx)rD   rQ   r   rE   s      r6   r?   zPlamoDecoder.__init__{  sr     6334
4 fx/BC4
 #00qd''1,
s   A7r   c                 ,   |d gt        | j                        z  }t        ||| j                           }| j                  t        ||| j                           }nd }t        | j                  |      D ]  \  }} |||j                  r|n||      }  |S )N)ry   rx   )lenr   r   r   r   r   zipr   )rD   r   rx   	attn_mask
mamba_masklcs          r6   rJ   zPlamoDecoder.__call__  s    =FS--E)!U4;;-?@	<<#(E$,,,?@JJ
 e$
#$::Z9A % r5   r   rN   s   @r6   r   r   z  s&    -y -T -"(( r5   r   c                   H     e Zd Zdef fdZ	 ddej                  fdZ xZS )
PlamoModelrQ   c                    t         |           || _        |j                  | _        t	        j
                  |j                  |j                        | _        t        |      | _	        t        |j                  |j                        | _        y )N)r9   )r>   r?   rQ   r*   rb   	Embeddingr   embed_tokensr   r   r8   r   normrv   s     r6   r?   zPlamoModel.__init__  sg     ++LL):):F<N<NO"6*F..F4G4GH	r5   inputsc                     |j                   \  }}| j                  |      }| j                  ||      }| j                  |      S r=   )r[   r   r   r   )rD   r   rx   r   
seq_lengthr   r   s          r6   rJ   zPlamoModel.__call__  sF    
 "(
Jf%kk

 yy~r5   r=   r   rN   s   @r6   r   r     s'    Iy I r5   r   c                        e Zd Zdeddf fdZdeeef   deeef   fdZd Zdde	j                  de	j                  fd	Zed
        Z xZS )ModelrQ   r;   Nc                    t         |           || _        |j                  | _        t	        |      | _        |j                  | _        |j                  s2t        j                  |j                  | j                  d      | _        y y )NFrS   )r>   r?   rQ   r   r   modelr*   r   rb   rc   r   lm_headrv   s     r6   r?   zModel.__init__  sj     ++'
 ++))&(ii""DOO%'DL *r5   weightsc                     |j                         D ]2  \  }}d|v s|j                  d   dk7  s|j                  dd      ||<   4 |S )Nzconv1d.weightr   r	   r$   )itemsr[   moveaxis)rD   r   r   r   s       r6   sanitizezModel.sanitize  sH    MMODAq!#q(8ZZ1-
 $ r5   c                     | j                   D cg c]$  }|j                  rt        d      n	t               & c}S c c}w )Nr$   )size)r   r   r   r   )rD   r   s     r6   
make_cachezModel.make_cache  s4     KO++V+Qqzz#wy@+VVVs   );r   c                     | j                  ||      }| j                  j                  r'| j                   j                  j	                  |      }|S | j                  |      }|S )N)r   rx   )r   rQ   r   r   	as_linearr   )rD   r   rx   outputslogitss        r6   rJ   zModel.__call__  s`    **  
 ;;**ZZ,,66w?F  \\'*Fr5   c                 B    | j                   j                  j                  S r=   )r   r   )rD   s    r6   r   zModel.layers  s    zz  '''r5   r=   )r+   r,   r-   r   r?   dictr   r   r  r@   rL   rJ   propertyr   rM   rN   s   @r6   r   r     sn    y T S#X 4S> W

rxx 
 
 ( (r5   r   )#mathdataclassesr   typingr   r   mlx.corecorer@   mlx.nnrb   mlx_lm.models.baser   r   r   activationsr
   rx   r   r   ssmr   r   Moduler8   rP   r   r   r   r0   r2   r   r   r   r   r4   r5   r6   <module>r     s     !     T T  '    ,
bii 
$b"BII b"JE#		 E#P4")) 4"3,		 3,l	?Y 	?3 	?4 	?299 B 8'(BII '(r5   