
    (HJj2                        d dl mZ d dlmZ d dlmZmZmZ d dlm	Z
 d dlmZ d dlmZmZmZ ddlmZmZmZ ddlmZ e G d	 d
e             Zed        Z G d dej2                        Z G d dej2                        Z G d dej2                        Z G d dej2                        Z G d dej2                        Z G d dej2                        Zy)    )	dataclass)	lru_cache)AnyListOptionalN)shard_inplaceshard_linearsum_gradients   )BaseModelArgscreate_attention_maskscaled_dot_product_attention)	SwitchGLUc                       e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed	<   eed
<   eed<   eed<   eed<   eed<   dZeed<   dZ	eed<   dZ
ee   ed<   dZeed<   y)	ModelArgs
model_typehidden_sizeintermediate_sizenum_attention_headsnum_key_value_headsmax_position_embeddingsnum_experts_per_toknum_local_expertsshared_intermediate_sizenum_hidden_layersrms_norm_eps
rope_theta
rotary_dim
vocab_sizeFtie_word_embeddingssigmoidscoring_funcNhead_dimTuse_qk_norm)__name__
__module____qualname__str__annotations__intfloatr    boolr"   r#   r   r$        _/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx_lm/models/minimax.pyr   r      s    O  !!OO %%!L#!"Hhsm"Kr.   r   c                 t     t         j                  d        t         j                   fd       fd}|S )Nc                 ~    | j                  t        j                        j                         j	                  dd      S )NT)keepdims)astypemxfloat32squaresum)xs    r/   _cast_square_sumz*sharded_rms_norm.<locals>._cast_square_sum'   s.    xx

#**,00d0CCr.   c                 4   t         j                  j                  |      }t        j                  || j                  d   j                         z  z  |z         }| j                  t         j                        |z  |z  j                  | j                        S )Ngroupr2   )	r5   distributedall_sumrsqrtshapesizer4   r6   dtype)r9   norm2wepsnormr=   s        r/   
_normalizez$sharded_rms_norm.<locals>._normalize+   ss    &&uE&:xxuzz|!;<sBC$t+a/77@@r.   c                 &     |  |       ||      S Nr-   )r9   rE   rF   r:   rH   s      r/   _inner_sharded_rms_normz1sharded_rms_norm.<locals>._inner_sharded_rms_norm4   s    !-a0!S99r.   )r5   compile)r=   rK   r:   rH   s   ` @@r/   sharded_rms_normrM   %   sB    ZZD D ZZA A: #"r.   c                        e Zd Z	 d
dededeej                  j                     f fdZ	d Z
d Zedddeej                  j                     fd	       Z xZS )ShardedRMSNormNdimsrF   r=   c                     t         |           |xs t        j                  j	                         }t        j
                  ||j                         z  f      | _        || _        || _	        y rJ   )
super__init__r5   r>   initonesrB   weightr=   rF   )selfrP   rF   r=   	__class__s       r/   rS   zShardedRMSNorm.__init__;   sS     	.,,.ggtuzz|356
r.   c                     | j                   j                  d   | j                  j                         z   d| j                   S )Nr   z, eps=)rV   rA   r=   rB   rF   rW   s    r/   _extra_reprzShardedRMSNorm._extra_reprD   s7    ++##A&)::;6$((LLr.   c                 V     t        | j                        || d   | j                        S )NrV   )rM   r=   rF   )rW   r9   s     r/   __call__zShardedRMSNorm.__call__G   s%    +

+AtH~txxHHr.   r<   c                    | |j                   j                  d   |j                  |      }t        j                  t        j
                  |j                   |j                         d      |j                                  |_         |S )Nr   r<   r2   axis)rV   rA   rF   r5   
contiguoussplitrB   rank)clsnorm_moduler=   sns       r/   from_rms_normzShardedRMSNorm.from_rms_normJ   se     ##))!,kooUKMMHH[''B?

M
	 	r.   )gh㈵>N)r%   r&   r'   r*   r+   r   r5   r>   GrouprS   r[   r]   classmethodrg   __classcell__rX   s   @r/   rO   rO   :   sl    TX#3;BNN<P<P3QMI EI$,R^^-A-A$B r.   rO   c            	            e Zd Zdef fdZ	 	 ddej                  deej                     dee   dej                  fdZ	 xZ
S )	MiniMaxAttentionargsc                    t         |           |j                  x| _        }|j                  | _        |j
                  | _        |j                  xs ||j                  z  x| _        }|dz  | _        t        j                  |j                  | j                  |z  d      | _
        t        j                  |j                  | j
                  |z  d      | _        t        j                  |j                  | j
                  |z  d      | _        t        j                  | j                  |z  |j                  d      | _        t        |d      r|j                  nd| _        | j                  rft        j                   || j                  z  |j"                        | _        t        j                   || j
                  z  |j"                        | _        t        j(                  |j*                  d|j,                        | _        y )Ng      Fbiasr$   rF   )traditionalbase)rR   rS   r   
hidden_dimr   r   r#   scalennLinearq_projk_projv_projo_projhasattrr$   RMSNormr   q_normk_normRoPEr   r   rope)rW   rn   r   r#   rX   s       r/   rS   zMiniMaxAttention.__init__W   s   (,(8(88+#'#;#; #'#;#; MMD[D,D,DD	
 t^
iid66A
 iid66A
 iid66A
 ii$$x/1A1A
 07t]/K4++QV**43339J9JDK **43339J9JDK GGDOOT__U	r.   r9   maskcachereturnc                    |j                   \  }}}| j                  |      | j                  |      | j                  |      }	}}| j                  r"| j                  |      }| j                  |      }|j                  ||| j                  d      j                  dddd      }|j                  ||| j                  d      j                  dddd      }|	j                  ||| j                  d      j                  dddd      }	|P| j                  ||j                        }| j                  ||j                        }|j                  ||	      \  }}	n"| j                  |      }| j                  |      }t        |||	|| j                  |      }
|
j                  dddd      j                  ||d      }
| j!                  |
      S )Nr2   r      r      )offset)r   rv   r   )rA   ry   rz   r{   r$   r   r   reshaper   	transposer   r   r   update_and_fetchr   rv   r|   )rW   r9   r   r   BLDquerieskeysvaluesoutputs              r/   r]   zMiniMaxAttention.__call__{   s    ''1a $AAAvkk'*G;;t$D//!Q(@(@"EOOq!Q
 ||Aq$":":B?II!QPQSTU1d&>&>CMMq!Q
 iii=G99T%,,97D 11$?LD&ii(G99T?D-T6djjt
 !!!Q1-55aB?{{6""r.   NNr%   r&   r'   r   rS   r5   arrayr   r   r]   rj   rk   s   @r/   rm   rm   V   sZ    "VY "VN $(#	$#88$# rxx $# }	$#
 
$#r.   rm   c                   \     e Zd Zdef fdZdej                  dej                  fdZ xZS )MiniMaxSparseMoeBlockrn   c                 ^   t         |           |j                  | _        t        j                  |j
                  |j                  d      | _        t        |j
                  |j                  |j                        | _
        t        j                  |j                  f      | _        d | _        y NFrp   )rR   rS   r   rw   rx   r   r   gater   r   
switch_mlpr5   zerose_score_correction_biassharding_grouprW   rn   rX   s     r/   rS   zMiniMaxSparseMoeBlock.__init__   s    #'#;#; IId..0F0FUS	#d44d6L6L
 (*xx1G1G0I'J$"r.   r9   r   c                    | j                    t        | j                         |      }| j                  |j                  t        j
                              }t	        j                  |      }|}|| j                  z   }| j                  }t	        j                  | |dz
  d      dd |f   }t	        j                  ||d      }|t	        j                  |dd      dz   z  }|j                  |j                        }| j                  ||      }||d	   z  j                  d
      }| j                   +t        j                  j                  || j                         }|S )Nr   r2   )kthr`   .r_   T)r`   r3   g#B;).Nr<   )r   r
   r   r4   r5   r6   r!   r   r   argpartitiontake_along_axisr8   rC   r   r>   r?   )rW   r9   gatesscoresorig_scoreskindsys           r/   r]   zMiniMaxSparseMoeBlock.__call__   s2   *2d11215A		!((2::./E"$666$$wAE;C!GD##KB?266&rDAEIJqww'OOAt$	""''R'0*&&q0C0C&DAr.   )	r%   r&   r'   r   rS   r5   r   r]   rj   rk   s   @r/   r   r      s*    	#Y 	#"(( rxx r.   r   c            	            e Zd Zdef fdZ	 	 ddej                  deej                     dee   dej                  fdZ	 xZ
S )	MiniMaxDecoderLayerrn   c                 "   t         |           t        |      | _        t	        |      | _        t        j                  |j                  |j                        | _
        t        j                  |j                  |j                        | _        y )Nrr   )rR   rS   rm   	self_attnr   block_sparse_moerw   r~   r   r   input_layernormpost_attention_layernormr   s     r/   rS   zMiniMaxDecoderLayer.__init__   sh    )$/ 5d ;!zz$*:*:@Q@QR(*

$"3"3)
%r.   r9   r   r   r   c                     || j                  | j                  |      ||      z   }|| j                  | j                  |            z   }|S rJ   )r   r   r   r   )rW   r9   r   r   rs        r/   r]   zMiniMaxDecoderLayer.__call__   sK     t33A6eDD%%d&C&CA&FGGr.   r   r   rk   s   @r/   r   r      sW    

Y 

 $(#	88 rxx  }	
 
r.   r   c            	            e Zd Zdef fdZ	 	 ddej                  deej                     dee   dej                  fdZ	 xZ
S )	MiniMaxModelrn   c                 N   t         |           t        j                  |j                  |j
                        | _        t        |j                        D cg c]  }t        |       c}| _
        t        j                  |j
                  |j                        | _        y c c}w )N)rn   rr   )rR   rS   rw   	Embeddingr   r   embed_tokensranger   r   layersr~   r   rG   )rW   rn   _rX   s      r/   rS   zMiniMaxModel.__init__   s    LL$:J:JK 5:$:P:P4Q
4QqT*4Q
 JJt//T5F5FG		
s   B"inputsr   r   r   c                     | j                  |      }|d gt        | j                        z  }t        ||d         }t	        | j                  |      D ]  \  }} ||||      } | j                  |      S )Nr   )r   lenr   r   ziprG   )rW   r   r   r   hlayercs          r/   r]   zMiniMaxModel.__call__   su     f%=FS--E$Qa1DKK/HE1aq!A 0 yy|r.   r   r   rk   s   @r/   r   r      sY    HY H $(#	 rxx  }	
 
r.   r   c                        e Zd Zdef fdZ	 	 ddej                  deej                     dee   fdZ	d Z
ddeej                  j                     fd	Zed
        Zed        Zed        Z xZS )Modelrn   c                     t         |           || _        |j                  | _        t	        |      | _        |j                  s2t        j                  |j                  |j                  d      | _        y y r   )rR   rS   rn   r   r   modelr    rw   rx   r   r   lm_headr   s     r/   rS   zModel.__init__   sX    	//!$'
''99T%5%5tUSDL (r.   r   r   r   c                     | j                  |||      }| j                  j                  r'| j                   j                  j	                  |      }|S | j                  |      }|S )N)r   r   r   )r   rn   r    r   	as_linearr   )rW   r   r   r   outs        r/   r]   zModel.__call__  s[     jjTj?99((**))33C8C 
 ,,s#C
r.   c                 .   d }i }|j                         D ];  \  }}d|v r(|}|j                  dd      }||   } |||      }|||<   2||vs7|||<   = |}d|vr|S t        | j                  j                        D ]  }	d|	 }
ddd	d
}|j                         D ]r  \  }}|
 d| d|v st        | j                  j
                        D cg c]  }|j                  |
 d| d| d       }}t        j                  |      ||
 d| d<   t  |S c c}w )z3Dequantize FP8 weights and restructure MoE experts.c                    t         j                  }t        j                  | t         j                        } d}| j                  \  }}| |z  }| |z  }t        j                  | d|fd|ff      } | j                  ||z   |z  |||z   |z  |f      } | |d d d d d d f   z  j                  ||z   ||z         } | d |d |f   j                  |      S )N)rC      r   )r5   bfloat16from_fp8rA   padr   r4   )rV   	scale_invrC   bsmn
pad_bottompad_sides           r/   dequantzModel.sanitize.<locals>.dequant  s    KKE[[r{{;FB<<DAq"JbyHVVFa_q(m$DEF^^j.R'a(lr-A2FF yD!T)9::CCJHF "1"bqb&>((//r.   weight_scale_inv
_scale_inv z3model.layers.0.block_sparse_moe.experts.0.w1.weightzmodel.layers.	gate_proj	down_projup_proj)w1w2w3z.block_sparse_moe.experts.0.z.weightz.block_sparse_moe.experts..z.block_sparse_moe.switch_mlp.)	itemsreplacer   rn   r   r   popr5   stack)rW   weightsr   new_weightsr   vr   wkrV   lprefixmapping	orig_namenew_nameeto_joins                   r/   sanitizezModel.sanitize  sn   	0" MMODAq!Q&	YY|R0  3"(B+%!"A $  AONtyy223A$QC(F(9MG'.}}#	8X9)GLPWW
 "'tyy'B'B!C	 "DA  %h&@1YKwW "D	   ) !("?zQ (7 4 s   !Dr=   c                    |xs t         j                  j                         }|j                         }|j	                         }| j
                  j                  D ]$  }t        |j                  j                  d|      |j                  _	        t        |j                  j                  d|      |j                  _
        t        |j                  j                  d|      |j                  _        t        |j                  j                  d|      |j                  _        |j                  j                  rtt        j                  |j                  j                   |      |j                  _        t        j                  |j                  j"                  |      |j                  _        |j                  xj$                  |z  c_        |j                  xj&                  |z  c_        t)        |j*                  j,                  j.                  d|       t)        |j*                  j,                  j0                  d|       t)        |j*                  j,                  j2                  d|       ||j*                  _        ' y )Nzall-to-shardedr<   zsharded-to-all)r5   r>   rT   rB   rc   r   r   r	   r   ry   rz   r{   r|   r$   rO   rg   r   r   r   r   r   r   r   r   r   r   r   )rW   r=   Nrc   r   s        r/   shardzModel.shardI  s   .,,.JJLzz|ZZ&&E%1&&(8&EOO" &2&&(8&EOO" &2&&(8&EOO" &2&&(8&EOO" **)7)E)EOO**% *F *& *8)E)EOO**% *F *& OO//A5/OO//A5/ &&11;; 
 &&11;; 
 &&1199 
 5:E""1S 'r.   c                 .    | j                   j                  S rJ   )r   r   rZ   s    r/   r   zModel.layersx  s    zz   r.   c                     d }|S )Nc                 
    d| vS )Nr   r-   )r   s    r/   	predicatez'Model.cast_predicate.<locals>.predicate~  s    ,A55r.   r-   rW   r   s     r/   cast_predicatezModel.cast_predicate|  s    	6 r.   c                     d }|S )Nc                 0    | j                  d      rdddS y)Nzblock_sparse_moe.gate@      )
group_sizebitsT)endswith)pathr   s     r/   r   z(Model.quant_predicate.<locals>.predicate  s    }}45&(!44r.   r-   r   s     r/   quant_predicatezModel.quant_predicate  s    	
 r.   r   rJ   )r%   r&   r'   r   rS   r5   r   r   r   r]   r   r>   rh   r   propertyr   r   r  rj   rk   s   @r/   r   r      s    TY T $(#	 rxx  }	3j-:8BNN$8$89 -:^ ! !    r.   r   ) dataclassesr   	functoolsr   typingr   r   r   mlx.corecorer5   mlx.nnrw   mlx.nn.layers.distributedr   r	   r
   rt   r   r   r   switch_layersr   r   rM   ModulerO   rm   r   r   r   r   r-   r.   r/   <module>r     s    "  & &   P P T T $   * # #(RYY 8I#ryy I#X#BII #L")) 0299 <LBII Lr.   