
    (HJj                     ^   d dl mZ d dlmZmZmZmZ d dlmZ	 d dl
mZ ddlmZ ddlmZmZmZ ddlmZ e G d d	e             Z G d
 dej*                        Z G d dej*                        Z G d dej*                        Z G d dej*                        Z G d dej*                        Zy)    )	dataclass)AnyDictOptionalUnionN   )swiglu)BaseModelArgscreate_attention_maskscaled_dot_product_attention)initialize_ropec                       e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed	<   eed
<   eed<   eed<   eed<   eed<   eed<   eed<   dZe	e
eeeef   f      ed<   dZeed<   y)	ModelArgs
model_typehidden_sizenum_hidden_layersintermediate_sizenum_attention_headsrms_norm_eps
vocab_sizelogits_scalingattention_multiplierembedding_multiplierresidual_multipliermax_position_embeddingsnum_key_value_headsattention_biasmlp_bias
rope_thetaNrope_scalingTtie_word_embeddings)__name__
__module____qualname__str__annotations__intfloatboolr    r   r   r   r!        _/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx_lm/models/granite.pyr   r      s    OO  N;?L(4U5#:%6 678? $$r+   r   c            	            e Zd Zdef fdZ	 	 ddej                  deej                     dee   dej                  fdZ	 xZ
S )		Attentionargsc                 d   t         |           |j                  }|j                  x| _        }|j
                  x| _        }|j                  |z  x| _        }|j                  | _	        |j                  }t        j                  |||z  |      | _        t        j                  |||z  |      | _        t        j                  |||z  |      | _        t        j                  ||z  ||      | _        t#        | j                  |j$                  d|j&                  |j(                        | _        y )NbiasF)super__init__r   r   n_headsr   
n_kv_headshead_dimr   scaler   nnLinearq_projk_projv_projo_projr   r   r    r   rope)selfr/   dimr5   r6   r7   r   	__class__s          r,   r4   zAttention.__init__%   s    !%!9!99w'+'?'??*#'#3#3w#>>..
,,iiWx%7nMiiZ(%:PiiZ(%:Pii( 2CnM#MMOO((
	r+   xmaskcachereturnc                 2   |j                   \  }}}| j                  |      | j                  |      | j                  |      }	}}|j	                  ||| j
                  d      j                  dddd      }|j	                  ||| j                  d      j                  dddd      }|	j	                  ||| j                  d      j                  dddd      }	|P| j                  ||j                        }| j                  ||j                        }|j                  ||	      \  }}	n"| j                  |      }| j                  |      }t        |||	|| j                  |      }
|
j                  dddd      j	                  ||d      }
| j                  |
      S )Nr      r      )offset)rE   r8   rD   )shaper;   r<   r=   reshaper5   	transposer6   r?   rK   update_and_fetchr   r8   r>   )r@   rC   rD   rE   BLDquerieskeysvaluesoutputs              r,   __call__zAttention.__call__=   si    ''1a $AAAv //!Qb9CCAq!QO||Aq$//26@@Aq!L1door:DDQ1aPiii=G99T%,,97D 11$?LD&ii(G99T?D-T6djjt
 !!!Q1-55aB?{{6""r+   NNr"   r#   r$   r   r4   mxarrayr   r   rW   __classcell__rB   s   @r,   r.   r.   $   sW    
Y 
6 $(#	#88# rxx # }	#
 
#r+   r.   c                   D     e Zd Zdef fdZdej                  fdZ xZS )MLPr/   c                 6   t         |           |j                  }|j                  }t	        |d      r|j
                  }nd}t        j                  |||      | _        t        j                  |||      | _	        t        j                  |||      | _
        y )Nr   Fr1   )r3   r4   r   r   hasattrr   r9   r:   	gate_proj	down_projup_proj)r@   r/   rA   
hidden_dimr   rB   s        r,   r4   zMLP.__init__]   sx    ++
4$}}HH3
B:sByyjx@r+   rF   c                 t    | j                  t        | j                  |      | j                  |                  S N)rc   r	   rb   rd   )r@   rC   s     r,   rW   zMLP.__call__k   s)    ~~fT^^A%6QHIIr+   	r"   r#   r$   r   r4   rZ   r[   rW   r\   r]   s   @r,   r_   r_   \   s#    AY AJRXX Jr+   r_   c            	            e Zd Zdef fdZ	 	 ddej                  deej                     dee   dej                  fdZ	 xZ
S )	TransformerBlockr/   c                    t         |           |j                  | _        |j                  | _        t	        |      | _        t        |      | _        t        j                  |j                  |j                        | _        t        j                  |j                  |j                        | _        |j                  | _        y )Neps)r3   r4   r   r   r.   	self_attnr_   mlpr9   RMSNormr   input_layernormpost_attention_layernormr   r@   r/   rB   s     r,   r4   zTransformerBlock.__init__p   s    #'#;#; ++"4t9!zz$*:*:@Q@QR(*

$"3"3)
% $(#;#; r+   rC   rD   rE   rF   c                     | j                  | j                  |      ||      }||| j                  z  z   }| j                  | j	                  |            }||| j                  z  z   }|S rg   )rn   rq   r   ro   rr   )r@   rC   rD   rE   rhouts          r,   rW   zTransformerBlock.__call__|   si     NN4//2D%@D,,,,HHT22156!d....
r+   rX   rY   r]   s   @r,   rj   rj   o   sW    
<Y 
< $(#	
88
 rxx 
 }	

 

r+   rj   c                   H     e Zd Zdef fdZ	 ddej                  fdZ xZS )GraniteModelr/   c                    t         |           || _        |j                  | _        |j                  | _        | j                  dkD  sJ t        j                  |j                  |j                        | _        t        |j                        D cg c]  }t        |       c}| _        t        j                  |j                  |j                        | _        |j                  | _        y c c}w )Nr   )r/   rl   )r3   r4   r/   r   r   r9   	Embeddingr   embed_tokensrangerj   layersrp   r   normr   )r@   r/   _rB   s      r,   r4   zGraniteModel.__init__   s    	//!%!7!7"""LL$:J:JK16t7M7M1N
1NA$'1N
 JJt//T5F5FG	$($=$=!	
s   C-inputsc                    | j                  |      | j                  z  }|d gt        | j                        z  }t	        ||d         }t        | j                  |      D ]  \  }} ||||      } | j                  |      S )Nr   )rE   )r|   r   lenr~   r   zipr   )r@   r   rE   rv   rD   layercs          r,   rW   zGraniteModel.__call__   s    
 f%(A(AA=FS--E$Qa1DKK/HE1aQ'A 0 yy|r+   rg   rh   r]   s   @r,   ry   ry      s%    >Y >  r+   ry   c                   X     e Zd Zdef fdZ	 ddej                  fdZed        Z	 xZ
S )Modelr/   c                    t         |           || _        |j                  | _        t	        |      | _        |j                  s1t        j                  |j                  |j                  d      | _        |j                  | _        y )NFr1   )r3   r4   r/   r   ry   modelr!   r9   r:   r   r   lm_headr   rs   s     r,   r4   zModel.__init__   sa    	//!$'
''99T%5%5tUSDL"11r+   r   c                     | j                  ||      }| j                  j                  r&| j                   j                  j	                  |      }n| j                  |      }|| j                  z  S rg   )r   r/   r!   r|   	as_linearr   r   )r@   r   rE   rw   s       r,   rW   zModel.__call__   sZ    
 jj'99((**))33C8C,,s#CT((((r+   c                 .    | j                   j                  S rg   )r   r~   )r@   s    r,   r~   zModel.layers   s    zz   r+   rg   )r"   r#   r$   r   r4   rZ   r[   rW   propertyr~   r\   r]   s   @r,   r   r      s9    2Y 2 
)
) ! !r+   r   )dataclassesr   typingr   r   r   r   mlx.corecorerZ   mlx.nnr9   activationsr	   baser
   r   r   
rope_utilsr   r   Moduler.   r_   rj   ry   r   r*   r+   r,   <module>r      s    " - -    T T ' % % %*5#		 5#pJ")) J&ryy 4299 @!BII !r+   