
    (HJj'                        d dl Z d dlmZ d dlmZ d dlmZmZ d dlm	Z
 d dlmZ ddlmZmZmZ e G d de             Z ee
j$                  d	
      d        Zd Z G d dej*                        Z G d dej*                        Z G d dej*                        Z G d dej*                        Z G d dej*                        Zy)    N)	dataclass)partial)AnyOptional   )BaseModelArgscreate_attention_maskscaled_dot_product_attentionc                       e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed	<   eed
<   dZeed<   dZe	ed<   dZ
eed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   y)	ModelArgs
model_typehidden_sizedense_attention_every_n_layersff_intermediate_sizegegelu_limitnum_hidden_layersnum_attention_headslayer_norm_epsilon
vocab_sizenum_key_value_heads      ?mup_attn_multiplierTmup_use_scalingg      $@mup_embedding_multiplierg       @mup_width_multiplieri@B rope_embedding_baserope_position_scale@   blocksparse_block_size   blocksparse_num_local_blocks   blocksparse_vert_strideN)__name__
__module____qualname__str__annotations__intfloatr   r   boolr   r   r   r   r   r!   r#        a/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx_lm/models/phi3small.pyr   r      s    O$''O!$$ OT &*e*"%%%!((!$$"$C$(* #*#$S$r-   r   T)	shapelessc           	      J   t        j                  t        j                  |       | t        j                  | d |            } t        j                  t        j                  |      |t        j                  || |            }| t        j                  d| z        z  }||dz   z  S )N)a_mina_maxgZd;?r   )mxwhereisinfclipsigmoid)a_gelua_linearlimitout_gelus       r.   gegelu_implr<   %   s    XX

d%0F
 xx

e4H
 

56>22Hx#~&&r-   c                 D    | dd d df   | ddd df   }}t        |||      S )N.   r   )r<   )xr:   r8   r9   s       r.   gegelur@   5   s4    cc{Ac14a4iLHFvx//r-   c            	            e Zd Zdef fdZd Zd Z	 	 d
dej                  de	ej                     de	e
   dej                  fd	Z xZS )	Attentionargsc                 :   t         |           |j                  }|j                  x| _        }|j
                  x| _        }||z  | _        |j                  |z  x| _        }t        j                  || j                  d| j                  z  z   |z        | _        t        j                  ||      | _        |j                  r||j                  z  }nt        j                   |      }d|z  | _        t        j$                  |d|j&                  |j(                        | _        ||j,                  z  dk(  rad| _        |j0                  | _        | j0                  dvrt3        d| j0                         |j4                  | _        |j6                  | _        y d| _        y )	Nr>   r   F)traditionalbasescaler   T)    r   zUnsupported block size )super__init__r   r   n_headsr   
n_kv_heads
n_q_per_kvhead_dimnnLinearquery_key_valuedenser   r   mathsqrtrG   RoPEr   r   roper   block_sparser   
ValueErrorr!   r#   )	selfrC   	layer_idxdimrK   rL   rN   norm_factor	__class__s	           r.   rJ   zAttention.__init__;   sy   !%!9!99w'+'?'??*!Z/#'#3#3w#>>!yy$,,T__!44@ 
 YYsC(
"T%=%==K))H-K;&
GG))**	
	 t:::a? $D*.*E*ED'**(: -d.I.I-JK  150Q0QD-+/+G+GD( %Dr-   c                    | j                   }| j                  }| j                  }| j                  }||z   dz
  |z  }||z   dz
  |z  }t	        j
                  ||z
  |      d d d d f   }	t	        j
                  |      d   }
t	        j
                  |      d d d f   t	        j
                  d|dz         d d d f   z   |z  }|dk(  d d d d d f   }|	|
k\  |	|
z
  |k  |z  z  } |j                  | j                  | j                  g|j                  dd   }t	        j                  t	        j                  ||d      |d      }||d| d d |f   fS )Nr   NNr   )axis.)r#   r!   r   rK   r3   arangereshaperL   rM   shaperepeat)rY   q_lenkv_lenvert_stridelocal_blocks
block_sizerK   	kv_blocksq_blocksq_posk_posmask_vert_strided
block_mask
dense_masks                 r.   _block_sparse_maskzAttention._block_sparse_maskc   sy   228800
,,j(1,;	J&*z9		)h.	:4D=I		)$Z0 IIi q)BIIa1,Eag,NN /!3QaZ@unU]\)->>

 (Z''OOT__
/9/?/?/D

 YYIIj*26


 :cE67GVG&;<<<r-   c           
         ||z  }|j                   d   }|j                   d   }t        j                  ||| j                  | j                  |df      }t        j
                  |d      }t        j
                  |d      }| j                  ||j                   d         \  }}	|t        j                  |dd      z  }
||
|z   }
|
t        j                  |	t        j                  d|
j                        t        j                  t        d       |
j                              z   }
t        j                  |
dd      }
|
|z  }t        j                  ||| j                  |df      S )Nr   r>   ra   r`   infT)rb   precise)re   r3   rd   rL   rM   expand_dimsrs   swapaxesr4   arraydtyper*   softmaxrK   )rY   querieskeysvaluesrG   maskBLrq   rr   scoresoutputs               r.   _block_sparse_attentionz!Attention._block_sparse_attention~   s0   '/MM!MM!**Wq$//4??Ar&RS~~dA&* "&!8!8DJJrN!K
J2;;tR44 d]F"((FLL1288U5\M6<<3X
 
 FT:&
 zz&1dllAr":;;r-   r?   r   cachereturnc                 X   |j                   \  }}}| j                  |      }|j                  ||d| j                  dz   | j                        }|dd dd d f   j                  dd      }|ddd d f   }	|ddd d f   }
|j                  dddd      }|	j                  dddd      }	|
j                  dddd      }
|P| j                  ||j                  	      }| j                  |	|j                  	      }	|j                  |	|
      \  }	}
n"| j                  |      }| j                  |	      }	| j                  r!| j                  ||	|
| j                  |
      }nt        ||	|
|| j                  |      }|j                  dddd      j                  ||d      }| j                  |      S )Nra   r>   .r`   r   r      )offset)rG   r   )r   rG   r   )re   rQ   rd   rM   rN   flatten	transposerV   r   update_and_fetchrW   r   rG   r
   rR   )rY   r?   r   r   r   r   Dqkvr|   r}   r~   r   s               r.   __call__zAttention.__call__   s    ''1a""1%kk!QDOOa$7Gc3B3k"**2r23A:S"aZ ##Aq!Q/~~aAq)!!!Q1-iii=G99T%,,97D 11$?LD&ii(G99T?D11vTZZd 2 F 2vU$**4F !!!Q1-55aB?zz&!!r-   r_   )r$   r%   r&   r   rJ   rs   r   r3   ry   r   r   r   __classcell__r]   s   @r.   rB   rB   :   sc    &&Y &&P=6<H $(#	$"88$" rxx $" }	$"
 
$"r-   rB   c                   >     e Zd Z fdZdej
                  fdZ xZS )MLPc                     t         |           |j                  }|j                  }|j                  | _        t        j                  |d|z        | _        t        j                  ||      | _        y )Nr>   )	rI   rJ   r   r   r   rO   rP   up_proj	down_proj)rY   rC   r[   
hidden_dimr]   s       r.   rJ   zMLP.__init__   sZ    ..
 --yya*n5:s3r-   r   c                 n    | j                  |      }| j                  t        || j                              S N)r   r   r@   r   )rY   r?   s     r.   r   zMLP.__call__   s+    LLO~~fQ(9(9:;;r-   )r$   r%   r&   rJ   r3   ry   r   r   r   s   @r.   r   r      s    4<RXX <r-   r   c            	            e Zd Zdef fdZ	 	 ddej                  deej                     dee   dej                  fdZ	 xZ
S )	TransformerBlockrC   c                 v   t         |           |j                  | _        |j                  | _        t	        ||      | _        t        |      | _        t        j                  |j                  |j                        | _        t        j                  |j                  |j                        | _        || _        y )Neps)rI   rJ   r   r   rB   	self_attnr   mlprO   	LayerNormr   input_layernormpost_attention_layernormrC   )rY   rC   rZ   r]   s      r.   rJ   zTransformerBlock.__init__   s    #'#;#; ++"43t9!||$"9"9 
 )+'')
% 	r-   r?   r   r   r   c                     | j                  | j                  |      ||      }||z   }| j                  | j                  |            }||z   }|S r   )r   r   r   r   )rY   r?   r   r   rhouts          r.   r   zTransformerBlock.__call__   sQ     NN4//2D%@EHHT22156!e
r-   r_   )r$   r%   r&   r   rJ   r3   ry   r   r   r   r   r   s   @r.   r   r      sW    Y $ $(#	
88
 rxx 
 }	

 

r-   r   c                   H     e Zd Zdef fdZ	 ddej                  fdZ xZS )	Phi3ModelrC   c                    t         |           || _        |j                  | _        |j                  | _        | j                  dkD  sJ |j
                  | _        t        j                  |j                  |j                        | _	        t        |j                        D cg c]  }t        ||       c}| _        t        j                  |j                  |j                        | _        y c c}w )Nr   )rC   rZ   r   )rI   rJ   rC   r   r   r   rO   	Embeddingr   embed_tokensranger   layersr   r   final_layernorm)rY   rC   lr]   s      r.   rJ   zPhi3Model.__init__   s    	//!%!7!7"""(,(E(E%LL$:J:JK 4112
2 $!42
  "||$"9"9 
	
s   !C.inputsc                 $   | j                  |      }| j                  r| j                  |z  }|d gt        | j                        z  }t	        ||d   d      }t        | j                  |      D ]  \  }} ||||      } | j                  |      S )Nr   T)return_array)r   r   lenr   r	   zipr   )rY   r   r   r   r   layercs          r.   r   zPhi3Model.__call__  s    
 f%((--1A=FS--E$QatDDKK/HE1aq!A 0 ##A&&r-   r   )	r$   r%   r&   r   rJ   r3   ry   r   r   r   s   @r.   r   r      s%    
Y 
& ''r-   r   c                   ^     e Zd Zdef fdZ	 ddej                  fdZed        Z	d Z
 xZS )ModelrC   c           	          t         |           |j                  | _        t        |      | _        || _        |j                  | _        t        j                  g dt        t        dd            z         | _        y )N)i i i i i i i i  )rI   rJ   r   r   modelrC   r   r3   ry   listr   _dummy_tokenizer_ids)rY   rC   r]   s     r.   rJ   zModel.__init__  sa    //t_
	$($=$=!$&HH<5()*%
!r-   r   c                     | j                  ||      }| j                   j                  j                  |      }| j                  r|| j                  z  }t	        d       || j
                  <   |S )Nru   )r   r   	as_linearr   r*   r   )rY   r   r   r   s       r.   r   zModel.__call__"  sb    
 jj'jj%%//4$$111C*/,D%%&
r-   c                 .    | j                   j                  S r   )r   r   )rY   s    r.   r   zModel.layers.  s    zz   r-   c                 `    |j                         D ci c]  \  }}d|vs|| c}}S c c}}w )Nzself_attn.rotary_emb.inv_freq)items)rY   weightskvs       r.   sanitizezModel.sanitize2  s<     %]]_
,TQ0OWX0XAqD_
 	
 
s   **r   )r$   r%   r&   r   rJ   r3   ry   r   propertyr   r   r   r   s   @r.   r   r     s>    	
Y 	
 

 ! !
r-   r   )rS   dataclassesr   	functoolsr   typingr   r   mlx.corecorer3   mlx.nnrO   rF   r   r	   r
   r   compiler<   r@   ModulerB   r   r   r   r   r,   r-   r.   <module>r      s     !      T T % % %, 	t$' %'0
I"		 I"X<")) <ryy :"'		 "'J 
BII  
r-   