
    (HJju                     f   d dl mZ d dlmZmZmZmZmZmZ d dl	m
Z d dlmZ ddlmZ ddlmZmZmZ ddlmZ e G d d	e             Z G d
 dej.                        Z G d dej.                        Z G d dej.                        Z G d dej.                        Z G d dej.                        Zy)    )	dataclass)AnyDictListOptionalTupleUnionN   )swiglu)BaseModelArgscreate_attention_maskscaled_dot_product_attention)SuScaledRoPEc                       e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   dZee   ed	<   d
Z	eed<   dZ
eed<   dZeeeeeee   f   f      ed<   dZeed<   dZeed<   dZeed<   dZeed<   d Zy)	ModelArgs
model_typehidden_sizenum_hidden_layersintermediate_sizenum_attention_headsrms_norm_eps
vocab_sizeNnum_key_value_headsi'  
rope_thetaFrope_traditionalrope_scaling      ?partial_rotary_factori   max_position_embeddingsi    original_max_position_embeddingstie_word_embeddingsc                       j                    j                   _          j                  rKddh}t         fd|D              st	        d|        j                  d   dvrt        d       d  _        y y y )Nlong_factortypec              3   :   K   | ]  }|j                   v   y wN)r   ).0keyselfs     \/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx_lm/models/phi3.py	<genexpr>z*ModelArgs.__post_init__.<locals>.<genexpr>&   s     I=Csd///=s   zrope_scaling must contain keys )longropesulinearzt[WARNING] rope_scaling 'type' currently only supports 'linear', 'su', and 'longrope'; setting rope scaling to false.)r   r   r   all
ValueErrorprint)r)   required_keyss   ` r*   __post_init__zModelArgs.__post_init__    s    ##+'+'?'?D$*F3MI=II #B=/!RSS  (0LL K %)!	 M     )__name__
__module____qualname__str__annotations__intfloatr   r   r   r   boolr   r   r	   r   r   r   r    r!   r3    r4   r*   r   r      s    OO)-#-J"d"CGL(4U5$u++=%> >?@G#&5&#)S),0$c0 %%)r4   r   c            	            e Zd Zdef fdZ	 	 ddej                  deej                     dee   dej                  fdZ	 xZ
S )		Attentionargsc           	         t         	|           |j                  }|j                  x| _        }|j
                  J |j
                  x| _        }|j                  | _        |j                  |z  x| _        }|dz  | _	        ||z  d||z  z  z   }t        j                  ||d      | _        t        j                  ||z  |d      | _        t        ||j                  z        }|j                   r`|j                   d   dv rOt#        ||j$                  |j&                  |j(                  |j                   d   |j                   d   	      | _        y d
}|j                   rC|j                   d   dk(  r1t-        |j                   d   t.              sJ d|j                   d   z  }t        j0                  ||j2                  |j$                  |      | _        y )Ng         Fbiasr$   )r,   r-   short_factorr#   )baser   r    rE   r#   r   r.   factorr
   )traditionalrF   scale)super__init__r   r   n_headsr   
n_kv_headsr   head_dimrI   nnLinearqkv_projo_projr:   r   r   r   r   r   r    rope
isinstancer;   RoPEr   )
r)   r@   dimrL   rM   rN   op_sizerope_dim
rope_scale	__class__s
            r*   rK   zAttention.__init__1   s   !%!9!99w''333'+'?'??*!%!7!7#'#3#3w#>>t^
H$qJ,A'BB		#wU;ii( 2CeDx$"<"<<=!2!26!:>P!P$__(,(D(D151V1V!..~> --m<DI J  T%6%6v%>(%J!$"3"3H"=uEEE!2!28!<<
 11__ 	DIr4   xmaskcachereturnc                    |j                   \  }}}| j                  |      }| j                  | j                  z  }t	        j
                  |||| j                  | j                  z  z   gd      \  }	}
}|	j                  ||| j                  d      j                  dddd      }	|
j                  ||| j                  d      j                  dddd      }
|j                  ||| j                  d      j                  dddd      }|P| j                  |	|j                        }	| j                  |
|j                        }
|j                  |
|      \  }
}n"| j                  |	      }	| j                  |
      }
t        |	|
||| j                  |      }|j                  dddd      j                  ||d      }| j                  |      S )	Naxisr   rB   r
      )offset)r]   rI   r\   )shaperQ   rL   rN   mxsplitrM   reshape	transposerS   rd   update_and_fetchr   rI   rR   )r)   r[   r\   r]   BLDqkv	query_posquerieskeysvaluesoutputs                r*   __call__zAttention.__call__W   s    ''1ammALL4==0	 ")Y4==)HHIPR!
v
 //!Qb9CCAq!QO||Aq$//26@@Aq!L1door:DDQ1aPiii=G99T%,,97D 11$?LD&ii(G99T?D-T6djjt
 !!!Q1-55aB?{{6""r4   NNr5   r6   r7   r   rK   rf   arrayr   r   rt   __classcell__rZ   s   @r*   r?   r?   0   sX    $Y $R $(#	#88# rxx # }	#
 
#r4   r?   c                   >     e Zd Z fdZdej
                  fdZ xZS )MLPc                     t         |           t        j                  |d|z  d      | _        t        j                  ||d      | _        y )NrB   FrC   )rJ   rK   rO   rP   gate_up_proj	down_proj)r)   rV   
hidden_dimrZ   s      r*   rK   zMLP.__init__z   s;    IIc1z>F:s?r4   r^   c                     | j                  |      }t        j                  |dd      \  }}| j                  t	        ||            S )NrB   r`   ra   )r}   rf   rg   r~   r   )r)   r[   gates      r*   rt   zMLP.__call__   s=    a ((1ab)a~~fT1o..r4   )r5   r6   r7   rK   rf   rw   rt   rx   ry   s   @r*   r{   r{   y   s    @
/RXX /r4   r{   c            	            e Zd Zdef fdZ	 	 ddej                  deej                     dee   dej                  fdZ	 xZ
S )	TransformerBlockr@   c                    t         |           |j                  | _        |j                  | _        t	        |      | _        t        |j                  |j                        | _        t        j                  |j                  |j                        | _        t        j                  |j                  |j                        | _        || _        y )Neps)rJ   rK   r   r   r?   	self_attnr{   r   mlprO   RMSNormr   input_layernormpost_attention_layernormr@   r)   r@   rZ   s     r*   rK   zTransformerBlock.__init__   s    #'#;#; ++"4t'')?)?@!zz$*:*:@Q@QR(*

$"3"3)
% 	r4   r[   r\   r]   r^   c                     | j                  | j                  |      ||      }||z   }| j                  | j                  |            }||z   }|S r&   )r   r   r   r   )r)   r[   r\   r]   rhouts          r*   rt   zTransformerBlock.__call__   sQ     NN4//2D%@EHHT22156!e
r4   ru   rv   ry   s   @r*   r   r      sW    
Y 
 $(#	
88
 rxx 
 }	

 

r4   r   c                   H     e Zd Zdef fdZ	 ddej                  fdZ xZS )	Phi3Modelr@   c                    t         |           || _        |j                  | _        |j                  | _        | j                  dkD  sJ t        j                  |j                  |j                        | _        t        |j                        D cg c]  }t        |       c}| _        t        j                  |j                  |j                        | _        y c c}w )Nr   )r@   r   )rJ   rK   r@   r   r   rO   	Embeddingr   embed_tokensranger   layersr   r   norm)r)   r@   _rZ   s      r*   rK   zPhi3Model.__init__   s    	//!%!7!7"""LL$:J:JK16t7M7M1N
1NA$'1N
 JJt//T5F5FG	
s   Cinputsc                     | j                  |      }|d gt        | j                        z  }t        ||d         }t	        | j                  |      D ]  \  }} ||||      } | j                  |      S )Nr   )r   lenr   r   zipr   )r)   r   r]   r   r\   layercs          r*   rt   zPhi3Model.__call__   su    
 f%=FS--E$Qa1DKK/HE1aq!A 0 yy|r4   r&   )	r5   r6   r7   r   rK   rf   rw   rt   rx   ry   s   @r*   r   r      s'    
HY 
H r4   r   c                   X     e Zd Zdef fdZ	 ddej                  fdZed        Z	 xZ
S )Modelr@   c                     t         |           |j                  | _        t        |      | _        |j
                  s1t        j                  |j                  |j                  d      | _
        || _        y )NFrC   )rJ   rK   r   r   modelr!   rO   rP   r   r   lm_headr@   r   s     r*   rK   zModel.__init__   sR    //t_
''99T%5%5tUSDL	r4   r   c                     | j                  ||      }| j                  j                  r'| j                   j                  j	                  |      }|S | j                  |      }|S r&   )r   r@   r!   r   	as_linearr   )r)   r   r]   r   s       r*   rt   zModel.__call__   sW    
 jj'99((**))33C8C 
 ,,s#C
r4   c                 .    | j                   j                  S r&   )r   r   )r)   s    r*   r   zModel.layers   s    zz   r4   r&   )r5   r6   r7   r   rK   rf   rw   rt   propertyr   rx   ry   s   @r*   r   r      s9    Y  

 ! !r4   r   )dataclassesr   typingr   r   r   r   r   r	   mlx.corecorerf   mlx.nnrO   activationsr   rF   r   r   r   
rope_utilsr   r   Moduler?   r{   r   r   r   r=   r4   r*   <module>r      s    " : :    T T $ ) ) )BF#		 F#R	/")) 	/ryy 4		 >!BII !r4   