
    (HJj                     J   d dl mZ d dlmZmZ d dlmZ d dlm	Z	 ddl
mZ ddlmZmZmZ e G d de             Z G d	 d
e	j"                        Z G d de	j"                        Z G d de	j"                        Z G d de	j"                        Z G d de	j"                        Zy)    )	dataclass)AnyOptionalN   )swiglu)BaseModelArgscreate_attention_maskscaled_dot_product_attentionc                       e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed	<   eed
<   eed<   eed<   dZeed<   dZ	eed<   y)	ModelArgs
model_typehidden_sizenum_hidden_layersintermediate_sizenum_attention_headsattention_biashead_dimrms_norm_eps
vocab_sizenum_key_value_headspartial_rotary_factor
rope_thetaTrope_traditionali   max_position_embeddingsN)
__name__
__module____qualname__str__annotations__intboolfloatr   r        \/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx_lm/models/glm4.pyr   r      s]    OMO  !d!#(S(r$   r   c                   D     e Zd Zdef fdZdej                  fdZ xZS )Glm4MLPargsc                     t         |           t        j                  |j                  d|j
                  z  d      | _        t        j                  |j
                  |j                  d      | _        y )N   Fbias)super__init__nnLinearr   r   gate_up_proj	down_projselfr(   	__class__s     r%   r.   zGlm4MLP.__init__    sZ    IIa$"8"88u
 4#9#94;K;KRWXr$   returnc                     | j                  |      }t        j                  |dd      \  }}| j                  t	        ||            S )Nr*   )axis)r1   mxsplitr2   r   )r4   xgate	up_statess       r%   __call__zGlm4MLP.__call__'   s>    a ((1ab1i~~fT9566r$   )	r   r   r   r   r.   r:   arrayr?   __classcell__r5   s   @r%   r'   r'      s!    YY Y7RXX 7r$   r'   c            	            e Zd Zdef fdZ	 ddej                  deej                     dee   dej                  fdZ	 xZ
S )	Glm4Attentionr(   c                    t         |           t        |d|j                  |j                  z        | _        |j                  | _        |j                  | _        | j
                  dz  | _	        t        j                  |j                  |j                  | j
                  z  |j                        | _        t        j                  |j                  |j                  | j
                  z  |j                        | _        t        j                  |j                  |j                  | j
                  z  |j                        | _        t        j                  |j                  | j
                  z  |j                  d      | _        t        j"                  t%        | j
                  |j&                  z        |j(                  |j*                        | _        y )Nr   g      r+   F)dimsbasetraditional)r-   r.   getattrr   r   r   n_headsr   
n_kv_headsscaler/   r0   r   q_projk_projv_projo_projRoPEr    r   r   r   roper3   s     r%   r.   zGlm4Attention.__init__.   se   *d..$2J2JJ
 //22]]D(
ii$$t}}4$$

 ii$$t}}4$$

 ii$$t}}4$$

 ii$$t}}4d6F6FU
 GGT]]T%?%??@--
	r$   r<   maskcacher6   c                 2   |j                   \  }}}| j                  |      | j                  |      | j                  |      }	}}|j	                  ||| j
                  d      j                  dddd      }|j	                  ||| j                  d      j                  dddd      }|	j	                  ||| j                  d      j                  dddd      }	|P| j                  ||j                        }| j                  ||j                        }|j                  ||	      \  }}	n"| j                  |      }| j                  |      }t        |||	|| j                  |      }
|
j                  dddd      j	                  ||d      }
| j                  |
      S )Nr8   r   r*   r      )offset)rT   rL   rS   )shaperM   rN   rO   reshaperJ   	transposerK   rR   rW   update_and_fetchr
   rL   rP   )r4   r<   rS   rT   BLDquerieskeysvaluesoutputs              r%   r?   zGlm4Attention.__call__P   sg    ''1a $AAAv//!Qb9CCAq!QO||Aq$//26@@Aq!L1door:DDQ1aPiii=G99T%,,97D 11$?LD&ii(G99T?D-T6djjt
 !!!Q1-55aB?{{6""r$   NNr   r   r   r   r.   r:   r@   r   r   r?   rA   rB   s   @r%   rD   rD   -   sQ     
Y  
F TX##!)"((!3#CKC=#	#r$   rD   c            	            e Zd Zdef fdZ	 ddej                  deej                     dee   dej                  fdZ	 xZ
S )	Glm4DecoderLayerr(   c                    t         |           t        |      | _        t	        |      | _        t        j                  |j                  |j                        | _
        t        j                  |j                  |j                        | _        t        j                  |j                  |j                        | _        t        j                  |j                  |j                        | _        y N)r(   )eps)r-   r.   rD   	self_attnr'   mlpr/   RMSNormr   r   input_layernormpost_attention_layernormpost_self_attn_layernormpost_mlp_layernormr3   s     r%   r.   zGlm4DecoderLayer.__init__l   s    &D14=!zz$*:*:@Q@QR(*

$"3"3)
% )+

$"3"3)
% #%**T-=-=4CTCT"Ur$   r<   rS   rT   r6   c                     || j                  | j                  | j                  |      ||            z   }|}| j                  | j	                  | j                  |                  |z   }|S N)ro   rj   rm   rp   rk   rn   )r4   r<   rS   rT   residuals        r%   r?   zGlm4DecoderLayer.__call__z   ss     --NN4//2D%@
 
 ##DHHT-J-J1-M$NO 	
 r$   rc   rd   rB   s   @r%   rf   rf   k   sR    VY V TX!)"((!3CKC=	r$   rf   c                   R     e Zd Zdef fdZ	 ddej                  dee   fdZ	 xZ
S )	Glm4Modelr(   c                 N   t         |           t        j                  |j                  |j
                        | _        t        |j                        D cg c]  }t        |       c}| _
        t        j                  |j
                  |j                        | _        y c c}w rh   )r-   r.   r/   	Embeddingr   r   embed_tokensranger   rf   layersrl   r   norm)r4   r(   _r5   s      r%   r.   zGlm4Model.__init__   s}    LL$:J:JK16t7M7M1N
1NA$'1N
 JJt//T5F5FG	
s   B"inputsrT   c                     | j                  |      }|d gt        | j                        z  }t        ||d         }t	        | j                  |      D ]  \  }} ||||      } | j                  |      S )Nr   )rT   )rx   lenrz   r	   zipr{   )r4   r}   rT   hrS   layercs          r%   r?   zGlm4Model.__call__   su    
 f%=FS--E$Qa1DKK/HE1aQ'A 0 yy|r$   rr   rd   rB   s   @r%   ru   ru      s5    HY H  $ }r$   ru   c                   b     e Zd Zdef fdZ	 ddej                  dee   fdZ	e
d        Z xZS )Modelr(   c                     t         |           || _        |j                  | _        t	        |      | _        t        j                  |j                  |j                  d      | _
        y )NFr+   )r-   r.   r(   r   ru   modelr/   r0   r   r   lm_headr3   s     r%   r.   zModel.__init__   sI    	//t_
yy!1!14??Or$   r}   rT   c                 H    | j                  ||      }| j                  |      S rr   )r   r   )r4   r}   rT   outs       r%   r?   zModel.__call__   s#    
 jj'||C  r$   c                 .    | j                   j                  S rr   )r   rz   )r4   s    r%   rz   zModel.layers   s    zz   r$   rr   )r   r   r   r   r.   r:   r@   r   r   r?   propertyrz   rA   rB   s   @r%   r   r      sI    PY P  $!! }! ! !r$   r   )dataclassesr   typingr   r   mlx.corecorer:   mlx.nnr/   activationsr   rG   r   r	   r
   r   Moduler'   rD   rf   ru   r   r#   r$   r%   <module>r      s    "      T T ) ) )"7bii 7;#BII ;#|ryy :		 6!BII !r$   