
    (HJj3                        d dl Z d dlmZ d dlmZ d dlmZ ddlm	Z	 d Z
ddZ G d dej                        Z G d	 d
ej                        Z G d dej                        Z G d dej                        Z G d dej                        Zy)    N)partial   swigluc                     |j                   ^ }}|j                         }t        j                  |      }t        j                  |      }| j                  dd      ||z     ||   |fS )Nr   )shapeflattenmxargsort)xindices_Morder	inv_orders         e/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx_lm/models/switch_layers.py_gather_sortr      s\    MMEQooGJJwE

5!I99QEQJ'BB    c                 B    | |   } |t        j                  | d|      } | S Nr   )r   	unflatten)r   r   r	   s      r   _scatter_unsortr      s(    	)ALLAu%Hr   c                   ~     e Zd Z	 	 	 	 ddededededededef fdZed	        Zed
        Z	ed        Z
ddZ xZS )QuantizedSwitchLinear
input_dimsoutput_dimsnum_expertsbias
group_sizebitsmodec           	         t         
|           t        j                  d|z        }t	        j
                  t        j                  j                  | ||||f      |||      ^| _        | _	        }	|	r|	d   nd | _
        |rt	        j                  ||f      | _        || _        || _        || _        | j!                          y )Nr   lowhighr	   )r    r!   r"   r   )super__init__mathsqrtr   quantizerandomuniformweightscalesbiaseszerosr   r    r!   r"   freeze)selfr   r   r   r   r    r!   r"   scaler0   	__class__s             r   r(   zQuantizedSwitchLinear.__init__   s     			!j.),.KKIIF"K<  
 "	-
)T[6 $*fQit+{!;<DI$		 	r   c                 N    | j                   j                  d   | j                  z  S N   )r/   r	   r    r3   s    r   r   z QuantizedSwitchLinear.input_dims?   s     {{  #doo55r   c                 4    | j                   j                  d   S Nr   r.   r	   r9   s    r   r   z!QuantizedSwitchLinear.output_dimsC       {{  ##r   c                 4    | j                   j                  d   S r   r<   r9   s    r   r   z!QuantizedSwitchLinear.num_expertsG   r=   r   c                     t        j                  || d   | d   | j                  d      |d| j                  | j                  | j
                  |
      }d| v r|t        j                  | d   |   d      z   }|S )Nr.   r/   r0   T)rhs_indices	transposer    r!   r"   sorted_indicesr   )r   
gather_qmmgetr    r!   r"   expand_dimsr3   r   r   rB   s       r   __call__zQuantizedSwitchLinear.__call__K   sw    MMNNHHX)
 T>BNN4<#8"==Ar   )T@      affineF)__name__
__module____qualname__intboolstrr(   propertyr   r   r   rH   __classcell__r5   s   @r   r   r      s     !! ! 	!
 ! ! ! !F 6 6 $ $ $ $r   r   c            	            e Zd Z	 ddedededef fdZed        Zed        Zed        Z	dd	Z
dd
ededefdZ xZS )SwitchLinearr   r   r   r   c                     t         |           t        j                  d|z        }t        j
                  j                  | ||||f      | _        |rt	        j                  ||f      | _	        y y )Nr   r$   )
r'   r(   r)   r*   r   r,   r-   r.   r1   r   )r3   r   r   r   r   r4   r5   s         r   r(   zSwitchLinear.__init__^   sn     			!j.)ii''Z8 ( 
 +{!;<DI r   c                 4    | j                   j                  d   S r7   r<   r9   s    r   r   zSwitchLinear.input_dimsl   r=   r   c                 4    | j                   j                  d   S r;   r<   r9   s    r   r   zSwitchLinear.output_dimsp   r=   r   c                 4    | j                   j                  d   S r   r<   r9   s    r   r   zSwitchLinear.num_expertst   r=   r   c                     t        j                  || d   j                  dd      ||      }d| v r|t        j                  | d   |   d      z   }|S )Nr.   rC   )r@   rB   r   )r   	gather_mmswapaxesrF   rG   s       r   rH   zSwitchLinear.__call__x   sY    LLN##B+)	
 T>BNN4<#8"==Ar   r    r!   r"   c           	         | j                   j                  \  }}}t        |||d|||      }t        j                  | j                   |||      ^|_         |_        }|r|d   nd |_        d| v r| j                  |_        |S )NF)r"   r   r   )r.   r	   r   r   r+   r/   r0   r   )	r3   r    r!   r"   r   r   r   qlr0   s	            r   to_quantizedzSwitchLinear.to_quantized   s    /3{{/@/@,[*"
 )+KKT)
%	29v "(F1IT	T>iiBG	r   )TrL   )rI   rJ   rK   )rM   rN   rO   rP   rQ   r(   rS   r   r   r   rH   rR   rb   rT   rU   s   @r   rW   rW   ]   s    PT==,/=>A=IM= $ $ $ $ $ $	s s c r   rW   c                   $     e Zd Z fdZd Z xZS )SwiGLUc                 "    t         |           y N)r'   r(   )r3   r5   s    r   r(   zSwiGLU.__init__   s    r   c                     t        ||      S rf   r   )r3   r   gates      r   rH   zSwiGLU.__call__   s    dAr   )rM   rN   rO   r(   rH   rT   rU   s   @r   rd   rd      s    r   rd   c            	       `     e Zd Z e       dfdedededef fdZdej                  fdZ	 xZ
S )		SwitchGLUFr   hidden_dimsr   r   c                     t         |           t        ||||      | _        t        ||||      | _        t        ||||      | _        || _        y N)r   )r'   r(   rW   	gate_projup_proj	down_proj
activationr3   r   rk   r   rq   r   r5   s         r   r(   zSwitchGLU.__init__   sO     	%j+{QUV#J[tT%k:{QUV$r   returnc                    t        j                  |d      }|j                  dk\  }|}d }|rt        ||      \  }}}| j                  rt        j
                  |      }| j                  |||      }| j                  |||      }| j                  | j                  ||      ||      }|rt        |||j                        }|j                  d      S N)rC   r   rI   )rB   rC   )r   rF   sizer   trainingstop_gradientro   rn   rp   rq   r   r	   squeeze)r3   r   r   do_sortidxr   x_upx_gates           r   rH   zSwitchGLU.__call__   s    NN1h' ,,"$	 ,Q 8AsI==""3'C||As7|;3w?NNOOD&)"  
 9gmm<Ayy}r   )rM   rN   rO   rd   rP   rQ   r(   r   arrayrH   rT   rU   s   @r   rj   rj      sH     8%% % 	% %bhh r   rj   c            	       x     e Zd Z ej                  d      dfdedededef fdZd	ej                  fd
Z
 xZS )	SwitchMLPprecise)approxFr   rk   r   r   c                     t         |           t        ||||      | _        t        ||||      | _        || _        y rm   )r'   r(   rW   fc1fc2rq   rr   s         r   r(   zSwitchMLP.__init__   s;     	
K4PZ4P$r   rs   c                    t        j                  |d      }|j                  dk\  }|}d }|rt        ||      \  }}}| j                  rt        j
                  |      }| j                  |||      }| j                  |      }| j                  |||      }|rt        |||j                        }|j                  d      S ru   )r   rF   rv   r   rw   rx   r   rq   r   r   r	   ry   )r3   r   r   rz   r{   r   s         r   rH   zSwitchMLP.__call__   s    NN1h' ,,"$	 ,Q 8AsI==""3'CHHQGH4OOAHHQGH49gmm<Ayy}r   )rM   rN   rO   nnGELUrP   rQ   r(   r   r~   rH   rT   rU   s   @r   r   r      sQ     277),%% % 	% %bhh r   r   rf   )r)   	functoolsr   mlx.corecorer   mlx.nnr   activationsr   r   r   Moduler   rW   rd   rj   r    r   r   <module>r      st        C?BII ?D8299 8vRYY '		 'T"		 "r   