
    (HJj                         d dl mZ d dlmZmZmZ d dlmZ d dl	m
Z
 ddlmZ ddlmZmZmZ ddlmZ e G d d	e             Z G d
 de
j(                        Z G d de
j(                        Z G d de
j(                        Z G d de
j(                        Z G d de
j(                        Z G d de
j(                        Zy)    )	dataclass)AnyListOptionalN   )swiglu)BaseModelArgscreate_attention_maskscaled_dot_product_attention)	SwitchGLUc                       e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   ee   ed<   eed<   eed	<   eed
<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   y)	ModelArgs
model_typehidden_sizenum_hidden_layersintermediate_sizenum_attention_headsattention_biasmlp_only_layersnum_expertsnum_experts_per_tokdecoder_sparse_stepn_shared_expertsmoe_intermediate_sizerms_norm_eps
vocab_sizenum_key_value_heads
rope_thetamax_position_embeddingsnorm_topk_probN)	__name__
__module____qualname__str__annotations__intboolr   float     ]/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx_lm/models/Klear.pyr   r      sp    O#YO  r*   r   c            	            e Zd Zdef fdZ	 	 ddej                  deej                     dee   dej                  fdZ	 xZ
S )	KlearAttentionargsc                 ,   t         |           |j                  | _        |j                  | _        |j                  |j                  z  | _        | j
                  dz  | _        t        j                  |j                  | j                  | j
                  z  |j                        | _
        t        j                  |j                  | j                  | j
                  z  |j                        | _        t        j                  |j                  | j                  | j
                  z  |j                        | _        t        j                  | j                  | j
                  z  |j                  |j                        | _        t        j                  | j
                  |j                        | _        t        j                  | j
                  |j                        | _        t        j$                  | j
                  d|j&                        | _        y )Ng      ࿩biasepsF)traditionalbase)super__init__r   r   r   head_dimscalennLinearr   q_projk_projv_projo_projRMSNormr   q_normk_normRoPEr   ropeselfr.   	__class__s     r+   r7   zKlearAttention.__init__%   s   #'#;#; #'#;#; ((D,D,DD]]D(
ii$$t}}4$$

 ii$$t}}4$$

 ii$$t}}4$$

 ii$$t}}4$$
 jjD4E4EFjjD4E4EFGGMM
	r*   xmaskcachereturnc                 n   |j                   \  }}}| j                  |      | j                  |      | j                  |      }	}}| j	                  |j                  ||| j                  d            j                  dddd      }| j                  |j                  ||| j                  d            j                  dddd      }|	j                  ||| j                  d      j                  dddd      }	|P| j                  ||j                        }| j                  ||j                        }|j                  ||	      \  }}	n"| j                  |      }| j                  |      }t        |||	|| j                  |      }
|
j                  dddd      j                  ||d      }
| j                  |
      S )Nr      r      )offset)rJ   r9   rI   )shaper<   r=   r>   rA   reshaper   	transposerB   r   rD   rP   update_and_fetchr   r9   r?   )rF   rH   rI   rJ   BLDquerieskeysvaluesoutputs              r+   __call__zKlearAttention.__call__K   s    ''1a $AAAv++OOAq$":":B?

)Aq!Q
 	 {{4<<1d.F.FKLVVq!Q
 1d&>&>CMMq!Q
 iii=G99T%,,97D 11$?LD&ii(G99T?D-T6djjt
 !!!Q1-55aB?{{6""r*   NNr!   r"   r#   r   r7   mxarrayr   r   r\   __classcell__rG   s   @r+   r-   r-   $   sX    $
Y $
R $(#	 #88 # rxx  # }	 #
 
 #r*   r-   c                   >     e Zd Z fdZdej
                  fdZ xZS )KlearMLPc                     t         |           t        j                  ||d      | _        t        j                  ||d      | _        t        j                  ||d      | _        y NFr0   )r6   r7   r:   r;   	gate_proj	down_projup_proj)rF   dim
hidden_dimrG   s      r+   r7   zKlearMLP.__init__o   sJ    3
?:s?yyju=r*   rK   c                 t    | j                  t        | j                  |      | j                  |                  S N)rh   r   rg   ri   )rF   rH   s     r+   r\   zKlearMLP.__call__u   s)    ~~fT^^A%6QHIIr*   )r!   r"   r#   r7   r_   r`   r\   ra   rb   s   @r+   rd   rd   n   s    >JRXX Jr*   rd   c                   \     e Zd Zdef fdZdej                  dej                  fdZ xZS )KlearSparseMoeBlockr.   c                 d   t         |           |j                  | _        |j                  | _        |j                  | _        t        j                  |j                  |j                  d      | _	        t        |j                  |j                  |j                        | _        t        |j                  |j                  |j                  z        | _        t        j                  |j                  d      | _        t#        j$                  | j                  ft"        j&                        | _        y )NFr0   )rk   rN   )dtype)r6   r7   r    r   r   top_kr:   r;   r   gater   r   expertsrd   r   shared_expertscoefficientr_   zerosfloat32expert_biasrE   s     r+   r7   zKlearSparseMoeBlock.__init__z   s    "11++--
IId..0@0@uM	 d88$:J:J
 '11D4I4II
 99T%5%5q988T%5%5$7rzzJr*   rH   rK   c                    t        j                  | j                  |      j                  t         j                              }|| j
                  j                  d      z   }| j                  }t        j                  | |dz
  d      dd |f   }t        j                  ||d      }| j                  r|t        j                  |dd      z  }|j                  |j                        }| j                  ||      }||d	   z  j                  d
      }t        j                  | j                  |      dd      }	| j!                  |      }
||	dd df   z  |
|	ddd f   z  z   }|S )N)r   r   rM   r   rM   )kthaxis.)r|   T)r|   keepdims).N)r|   precise)r_   sigmoidrs   astyperx   ry   rR   rr   argpartitiontake_along_axisr    sumrq   rt   softmaxrv   ru   )rF   rH   routing_weightsbiased_weightskindsscores
expert_out	y_expertscoefsharedys               r+   r\   zKlearSparseMoeBlock.__call__   s?   **TYYq\%8%8%DE(4+;+;+C+CJ+OOJJAECC!GL##OTCbffV"tDDFqww'\\!T*
&"3388b8A	zz$**1-BE$$Q'S"1"W%c12g(>>r*   )	r!   r"   r#   r   r7   r_   r`   r\   ra   rb   s   @r+   ro   ro   y   s,    KY K""(( rxx r*   ro   c            	            e Zd Zdedef fdZ	 	 d	dej                  deej                     dee	   dej                  fdZ
 xZS )
KlearDecoderLayerr.   	layer_idxc                    t         |           t        |      | _        ||j                  vr5|j
                  dkD  r&|dz   |j                  z  dk(  rt        |      | _        n%t        |j                  |j                        | _        t        j                  |j                  |j                        | _        t        j                  |j                  |j                        | _        y )Nr   r   r2   )r6   r7   r-   	self_attnr   r   r   ro   mlprd   r   r   r:   r@   r   input_layernormpost_attention_layernorm)rF   r.   r   rG   s      r+   r7   zKlearDecoderLayer.__init__   s    '-T111q i!mt7O7O%OST%T*40DH 0 0$2H2HIDH!zz$*:*:@Q@QR(*

$"3"3)
%r*   rH   rI   rJ   rK   c                     | j                  | j                  |      ||      }||z   }| j                  | j                  |            }||z   }|S rm   )r   r   r   r   )rF   rH   rI   rJ   rhouts          r+   r\   zKlearDecoderLayer.__call__   sQ     NN4//2D%@EHHT22156!e
r*   r]   )r!   r"   r#   r   r&   r7   r_   r`   r   r   r\   ra   rb   s   @r+   r   r      s^    
Y 
3 
& $(#	
88
 rxx 
 }	

 

r*   r   c                   j     e Zd Zdef fdZ	 ddej                  dee   dej                  fdZ	 xZ
S )
KlearModelr.   c                 P   t         |           t        j                  |j                  |j
                        | _        t        |j                        D cg c]  }t        ||       c}| _
        t        j                  |j
                  |j                        | _        y c c}w )N)r.   r   r2   )r6   r7   r:   	Embeddingr   r   embed_tokensranger   r   layersr@   r   norm)rF   r.   irG   s      r+   r7   zKlearModel.__init__   s    LL$:J:JK 4112
2 4152
 JJt//T5F5FG		
s   B#inputsrJ   rK   c                     | j                  |      }|d gt        | j                        z  }t        ||d         }t	        | j                  |      D ]  \  }} ||||      } | j                  |      S )Nr   )r   lenr   r
   zipr   )rF   r   rJ   r   rI   layercs          r+   r\   zKlearModel.__call__   su    
 f%=FS--E$Qa1DKK/HE1aq!A 0 yy|r*   rm   r^   rb   s   @r+   r   r      sC    HY H  $ } 
	r*   r   c                        e Zd Zdef fdZ	 d
dej                  dee   fdZ	d Z
ed        Zed        Zed	        Z xZS )Modelr.   c                     t         |           || _        |j                  | _        t	        |      | _        t        j                  |j                  |j                  d      | _
        y rf   )r6   r7   r.   r   r   modelr:   r;   r   r   lm_headrE   s     r+   r7   zModel.__init__   sJ    	//%
yy!1!14??Or*   r   rJ   c                 H    | j                  ||      }| j                  |      S rm   )r   r   )rF   r   rJ   r   s       r+   r\   zModel.__call__   s#    
 jj'||C  r*   c                 B   d|vr|S t        | j                  j                        D ]q  }d| d}dD ]d  }t        | j                  j                        D cg c]  }|j	                  | d| d| d       }}t        j                  |      || d| d<   f s |S c c}w )Nz-model.layers.0.mlp.experts.0.gate_proj.weightzmodel.layers.z.mlp.experts)rg   ri   rh   .z.weight)r   r.   r   r   popr_   stack)rF   weightslprefixnameestackeds          r+   sanitizezModel.sanitize   s    :'INtyy223A$QC|4F= #499#8#899 KK6(!A3avW =>9   68XXg5F6(!D612 > 4 s   !Bc                 .    | j                   j                  S rm   )r   r   )rF   s    r+   r   zModel.layers   s    zz   r*   c                     d }|S )Nc                 0    | j                  d      rdddS y)Nzmlp.gate@      )
group_sizebitsT)endswith)path_s     r+   	predicatez(Model.quant_predicate.<locals>.predicate   s    }}Z(&(!44r*   r)   rF   r   s     r+   quant_predicatezModel.quant_predicate   s    	
 r*   c                     d }|S )Nc                 
    d| vS )Nry   r)   )r   s    r+   r   z'Model.cast_predicate.<locals>.predicate  s     ))r*   r)   r   s     r+   cast_predicatezModel.cast_predicate  s    	* r*   rm   )r!   r"   r#   r   r7   r_   r`   r   r   r\   r   propertyr   r   r   ra   rb   s   @r+   r   r      sv    PY P  $!! }! ! !    r*   r   )dataclassesr   typingr   r   r   mlx.corecorer_   mlx.nnr:   activationsr   r5   r	   r
   r   switch_layersr   r   Moduler-   rd   ro   r   r   r   r)   r*   r+   <module>r      s    " & &    T T $   *G#RYY G#TJryy J "))  F		 < 81BII 1r*   