
    (HJj                     V   d dl mZ d dlmZmZ d dlmZ d dlm	Z	 ddl
mZ ddlmZmZmZ ddlmZ e G d d	e             Z G d
 de	j&                        Z G d de	j&                        Z G d de	j&                        Z G d de	j&                        Z G d de	j&                        Zy)    )	dataclass)AnyOptionalN   )swiglu)BaseModelArgscreate_attention_maskscaled_dot_product_attention)initialize_ropec                       e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   ee   ed<   eed<   eed	<   eed
<   eed<   eed<   eed<   y)	ModelArgshidden_sizeintermediate_size
model_typemax_position_embeddingsnum_attention_headsnum_key_value_headshead_dimnum_hidden_layersrms_norm_eps
vocab_size
rope_thetause_biastie_word_embeddingsN)	__name__
__module____qualname__int__annotations__strr   floatbool     `/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx_lm/models/ernie4_5.pyr   r      sQ    O  smONr$   r   c            	            e Zd Zdef fdZ	 	 ddej                  deej                     dee   dej                  fdZ	 xZ
S )		Attentionargsc                 n   t         |           |j                  }|j                  x| _        }|j
                  x| _        }|j                  xs ||z  x| _        }|dz  | _        t        j                  |||z  |j                        | _        t        j                  |||z  |j                        | _        t        j                  |||z  |j                        | _        t        j                  ||z  ||j                        | _        t!        ||j"                  d|j$                        | _        y )Ng      ࿩biasT)basetraditionalr   )super__init__r   r   n_headsr   
n_kv_headsr   scalennLinearr   q_projk_projv_projo_projr   r   r   rope)selfr(   dimr0   r1   r   	__class__s         r%   r/   zAttention.__init__    s    !%!9!99w'+'?'??*#'==#BC7NBt^
iiWx%7dmmLiiZ(%:OiiZ(%:Oii( 2CdmmL#$($@$@	
	r$   xmaskcachereturnc                 2   |j                   \  }}}| j                  |      | j                  |      | j                  |      }	}}|j	                  ||| j
                  d      j                  dddd      }|j	                  ||| j                  d      j                  dddd      }|	j	                  ||| j                  d      j                  dddd      }	|P| j                  ||j                        }| j                  ||j                        }|j                  ||	      \  }}	n"| j                  |      }| j                  |      }t        |||	|| j                  |      }
|
j                  dddd      j	                  ||d      }
| j                  |
      S )Nr      r      )offset)r?   r2   r>   )shaper5   r6   r7   reshaper0   	transposer1   r9   rE   update_and_fetchr
   r2   r8   )r:   r=   r>   r?   BLDquerieskeysvaluesoutputs              r%   __call__zAttention.__call__6   sg    ''1a $AAAv//!Qb9CCAq!QO||Aq$//26@@Aq!L1door:DDQ1aPiii=G99T%,,97D 11$?LD&ii(G99T?D-T6djjt
 !!!Q1-55aB?{{6""r$   NNr   r   r   r   r/   mxarrayr   r   rQ   __classcell__r<   s   @r%   r'   r'      sW    
Y 
2 $(#	#88# rxx # }	#
 
#r$   r'   c                   @     e Zd Zd fd	Zdej
                  fdZ xZS )MLPc                     t         |           t        j                  |||      | _        t        j                  |||      | _        t        j                  |||      | _        y )Nr*   )r.   r/   r3   r4   	gate_proj	down_projup_proj)r:   r;   
hidden_dimr   r<   s       r%   r/   zMLP.__init__T   sJ    3
B:sByyjx@r$   r@   c                 t    | j                  t        | j                  |      | j                  |                  S N)r\   r   r[   r]   )r:   r=   s     r%   rQ   zMLP.__call__Z   s)    ~~fT^^A%6QHIIr$   )F)r   r   r   r/   rT   rU   rQ   rV   rW   s   @r%   rY   rY   S   s    AJRXX Jr$   rY   c            	            e Zd Zdef fdZ	 	 ddej                  deej                     dee   dej                  fdZ	 xZ
S )	DecoderLayerr(   c                 b   t         |           t        |      | _        t	        |j
                  |j                  |j                        | _        t        j                  |j
                  |j                        | _        t        j                  |j
                  |j                        | _        y N)eps)r.   r/   r'   	self_attnrY   r   r   r   mlpr3   RMSNormr   input_layernormpost_attention_layernormr:   r(   r<   s     r%   r/   zDecoderLayer.__init___   sz    "4t'')?)?O!zz$*:*:@Q@QR(*

$"3"3)
%r$   r=   r>   r?   r@   c                     | j                  | j                  |      ||      }||z   }| j                  | j                  |            }||z   S r`   )rf   ri   rg   rj   )r:   r=   r>   r?   rhs         r%   rQ   zDecoderLayer.__call__i   sL     NN4//2D%@EHHT221561ur$   rR   rS   rW   s   @r%   rb   rb   ^   sW    
Y 
 $(#		88	 rxx 	 }		
 
	r$   rb   c                   H     e Zd Zdef fdZ	 ddej                  fdZ xZS )Ernie45Modelr(   c                 L   t         |           t        j                  |j                  |j
                        | _        t        |j                        D cg c]  }t        |       c}| _
        t        j                  |j
                  |j                        | _        y c c}w rd   )r.   r/   r3   	Embeddingr   r   embed_tokensranger   rb   layersrh   r   norm)r:   r(   _r<   s      r%   r/   zErnie45Model.__init__v   sw    LL$:J:JK389O9O3PQ3Pa|D)3PQJJt//T5F5FG	 Rs   B!inputsc                     | j                  |      }|d gt        | j                        z  }t        ||d         }t	        | j                  |      D ]  \  }} ||||      } | j                  |      S )Nr   )rs   lenru   r	   ziprv   )r:   rx   r?   rn   r>   layercs          r%   rQ   zErnie45Model.__call__|   su    
 f%=FS--E$Qa1DKK/HE1aq!A 0 yy|r$   r`   )	r   r   r   r   r/   rT   rU   rQ   rV   rW   s   @r%   rp   rp   u   s'    HY H r$   rp   c                   X     e Zd Zdef fdZ	 ddej                  fdZed        Z	 xZ
S )Modelr(   c                     t         |           || _        |j                  | _        t	        |      | _        |j                  s2t        j                  |j                  |j                  d      | _        y y )NFr*   )r.   r/   r(   r   rp   modelr   r3   r4   r   r   lm_headrk   s     r%   r/   zModel.__init__   sX    	//!$'
''99T%5%5tUSDL (r$   rx   c                     | j                  ||      }| j                  j                  r'| j                   j                  j	                  |      }|S | j                  |      }|S r`   )r   r(   r   rs   	as_linearr   )r:   rx   r?   outs       r%   rQ   zModel.__call__   sW    
 jj'99((**))33C8C 
 ,,s#C
r$   c                 .    | j                   j                  S r`   )r   ru   )r:   s    r%   ru   zModel.layers   s    zz   r$   r`   )r   r   r   r   r/   rT   rU   rQ   propertyru   rV   rW   s   @r%   r   r      s;    TY T 

 ! !r$   r   )dataclassesr   typingr   r   mlx.corecorerT   mlx.nnr3   activationsr   r,   r   r	   r
   
rope_utilsr   r   Moduler'   rY   rb   rp   r   r#   r$   r%   <module>r      s    "      T T '    1#		 1#hJ")) J299 .299 2!BII !r$   