
    (HJj                        d dl mZ d dlmZmZmZmZmZ d dlm	Z
 d dlmZ ddlmZ ddlmZmZmZ e G d de             Z	 	 dd	eeef   d
ee   deeeef      deeef   fdZ G d dej.                        Z G d dej.                        Z G d dej.                        Z G d dej.                        Z G d dej.                        Zy)    )	dataclass)AnyDictListOptionalUnionN   )swiglu)BaseModelArgscreate_attention_maskscaled_dot_product_attentionc                       e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed	<   d
Zeed<   d
Z	eed<   d
Z
eed<   dZeed<   dZeed<   y)	ModelArgs
model_typehead_dimnum_transformer_layers	model_dim
vocab_sizeffn_dim_divisornum_query_headsnum_kv_headsffn_multipliersTffn_with_glunormalize_qk_projectionsshare_input_output_layersgư>rms_norm_epsi'  rope_freq_constantN)__name__
__module____qualname__str__annotations__intr   r   boolr   r   r   floatr        _/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx_lm/models/openelm.pyr   r      si    OMNOL$%)d)&*t*L% %%r'   r   vdivisor	min_valuereturnc                 j    ||}t        |t        | |dz  z         |z  |z        }|d| z  k  r||z  }|S )a  
    This function is taken from the original tf repo.
    It ensures that all layers have a channel number that is divisible by the divisor
    It can be seen at:
    https://github.com/tensorflow/models/blob/2cfc99eff5e5eb729c6793d2f3d03aa1c9be2b15/research/slim/nets/mobilenet/mobilenet.py#L62
    Args:
        v: input value
        divisor: default to 8
        min_value: minimum divisor value
    Returns:
        new_v: new divisible value
       g?)maxr#   )r)   r*   r+   new_vs       r(   make_divisibler1      sL    " 		3q7Q;/7:WDEEsQwLr'   c            	            e Zd Zdedef fdZ	 	 d	dej                  deej                     dee	   dej                  fdZ
 xZS )
	Attentionargslayer_idc                    t         |           |j                  x| _        }|| _        |j                  x| _        }|j
                  |   x| _        }|j                  |   x| _        }|dz  | _	        ||dz  z   |z  }t        j                  ||d      | _        t        j                  ||z  |d      | _        |j                  | _        | j                  rLt        j                  ||j                         | _        t        j                  ||j                         | _        t        j&                  |d|j(                        | _        y )Ng      r.   Fbiaseps)traditionalbase)super__init__r   r5   r   r   n_headsr   
n_kv_headsscalennLinearqkv_projout_projr   RMSNormr   q_normk_normRoPEr   rope)	selfr4   r5   r   r   r?   r@   op_size	__class__s	           r(   r>   zAttention.__init__:   s   #'==0 %)^^3!%!5!5h!??w'+'8'8'BB*t^
j1n-9		)W5A		'H"4ieL(,(E(E%((**X43D3DEDK**X43D3DEDKGGH%d>U>UV	r'   xmaskcacher,   c                 <   |j                   \  }}}| j                  |      }|j                  ||| j                  | j                  dz  z   | j
                        j                  dddd      }t        j                  || j                  | j                  | j                  z   gd      \  }}	}
| j                  r"| j                  |      }| j                  |	      }	|P| j                  ||j                        }| j                  |	|j                        }	|j                  |	|
      \  }	}
n"| j                  |      }| j                  |	      }	t        ||	|
|| j                   |      }|j                  dddd      j                  ||d      }| j#                  |      S )	Nr.   r   r	      axis)offset)rP   rA   rO   )shaperD   reshaper?   r@   r   	transposemxsplitr   rG   rH   rJ   rU   update_and_fetchr   rA   rE   )rK   rN   rO   rP   BLDqkvquerieskeysvaluesoutputs               r(   __call__zAttention.__call__P   sp    ''1ammAkkq$,,$//A"56

)Aq!Q
 	 !#$,,t >?a!
v
 ((kk'*G;;t$Diii=G99T%,,97D 11$?LD&ii(G99T?D-T6djjt
 !!!Q1-55aB?}}V$$r'   NNr   r   r    r   r#   r>   rZ   arrayr   r   re   __classcell__rM   s   @r(   r3   r3   9   sa    WY W# W2 $(#	%%88%% rxx %% }	%%
 
%%r'   r3   c                   H     e Zd Zdedef fdZdej                  fdZ xZ	S )MLPr4   r5   c                 :   t         |           || _        |j                  }|j                  |   }t        t        ||j                  z  |j                              }t        j                  |d|z  d      | _
        t        j                  ||d      | _        y )N)r*   r.   Fr7   )r=   r>   r4   r   r   r#   r1   r   rB   rC   proj_1proj_2)rK   r4   r5   dimffn_multiplierintermediate_dimrM   s         r(   r>   zMLP.__init__y   s    	nn--h7/,,
 iiQ)9%9Fii 0#EBr'   r,   c                     | j                  |      }t        j                  |dd      \  }}| j                  t	        ||            S )Nr.   rV   rS   )rn   rZ   r[   ro   r
   )rK   rN   gates      r(   re   zMLP.__call__   s:    KKN((1ab)a{{6$?++r'   )
r   r   r    r   r#   r>   rZ   rh   re   ri   rj   s   @r(   rl   rl   x   s)    CY C# C ,RXX ,r'   rl   c            	            e Zd Zdedef fdZ	 	 d	dej                  deej                     dee	   dej                  fdZ
 xZS )
TransformerBlockr4   r5   c                    t         |           |j                  }t        ||      | _        t        ||      | _        t        j                  ||j                        | _
        t        j                  ||j                        | _        y )Nr5   r9   )r=   r>   r   r3   attnrl   ffnrB   rF   r   ffn_norm	attn_norm)rK   r4   r5   rp   rM   s       r(   r>   zTransformerBlock.__init__   sc    nndX6	th/

3D,=,=>CT->->?r'   rN   rO   rP   r,   c                     | j                  | j                  |      ||      }||z   }| j                  | j                  |            }||z   }|S N)ry   r|   rz   r{   )rK   rN   rO   rP   rhouts          r(   re   zTransformerBlock.__call__   sM     IIdnnQ'u5EHHT]]1%&!e
r'   rf   rg   rj   s   @r(   rv   rv      sa    @Y @# @ $(#	
88
 rxx 
 }	

 

r'   rv   c                   H     e Zd Zdef fdZ	 ddej                  fdZ xZS )OpenELMModelr4   c                    t         |           || _        |j                  | _        |j                  | _        | j                  dkD  sJ t        j                  |j                  |j                        | _        t        | j                        D cg c]  }t        ||       c}| _        t        j                  |j                  |j                        | _        y c c}w )Nr   rx   r9   )r=   r>   r4   r   r   rB   	Embeddingr   token_embeddingsrangerv   layersrF   r   norm)rK   r4   r5   rM   s      r(   r>   zOpenELMModel.__init__   s    	//&*&A&A#""" "T__dnn M "$"="=>
> TH5>
 JJt~~43D3DE		
s   Cinputsc                     | j                  |      }|d gt        | j                        z  }t        ||d         }t	        | j                  |      D ]  \  }} ||||      } | j                  |      S )Nr   )rP   )r   lenr   r   zipr   )rK   r   rP   r   rO   layercs          r(   re   zOpenELMModel.__call__   su    
 !!&)=FS--E$Qa1DKK/HE1aQ'A 0 yy|r'   r~   )	r   r   r    r   r>   rZ   rh   re   ri   rj   s   @r(   r   r      s'    FY F  r'   r   c                   X     e Zd Zdef fdZ	 ddej                  fdZed        Z	 xZ
S )Modelr4   c                     t         |           || _        |j                  | _        t	        |      | _        |j                  s2t        j                  |j                  |j                  d      | _        y y )NFr7   )r=   r>   r4   r   r   transformerr   rB   rC   r   r   lm_head)rK   r4   rM   s     r(   r>   zModel.__init__   sW    	//'---99T^^T__5QDL .r'   r   c                     | j                  ||      }| j                  j                  r'| j                   j                  j	                  |      }|S | j                  |      }|S r~   )r   r4   r   r   	as_linearr   )rK   r   rP   r   s       r(   re   zModel.__call__   s[    
 vu-99..""33==cBC 
 ,,s#C
r'   c                 .    | j                   j                  S r~   )r   r   )rK   s    r(   r   zModel.layers   s    &&&r'   r~   )r   r   r    r   r>   rZ   rh   re   propertyr   ri   rj   s   @r(   r   r      s;    RY R  ' 'r'   r   )   N)dataclassesr   typingr   r   r   r   r   mlx.corecorerZ   mlx.nnrB   activationsr
   r<   r   r   r   r   r%   r#   r1   Moduler3   rl   rv   r   r   r&   r'   r(   <module>r      s    " 3 3    T T & & && -1UCZc] eSj)* 5#:	4<%		 <%~,")) ,.ryy ,299 >'BII 'r'   