
    (HJj                          d dl Z d dlmZ d dlmZ ddlmZmZ  G d dej                        Z
 G d dej                        Z G d d	ej                        Zy)
    N   )QuantizedSwitchLinearSwitchLinearc                        e Zd Ze	 	 	 ddej
                  dededefd       Zdde	fdZ
	 	 	 	 dded	edededed
e	f fdZd Z xZS )
LoRALinearlinearrdropoutscalec                     | j                   j                  \  }}t        | t        j                        r|dz  | j
                  z  }t        |||||      }| |_        |S )N    )
input_dimsoutput_dimsr	   r
   r   )weightshape
isinstancennQuantizedLinearbitsr   r   )r   r	   r
   r   r   r   lora_lins          [/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx_lm/tuner/lora.py	from_basezLoRALinear.from_base   sa     #)--"5"5Zfb001#bFKK7J!#
 !    
dequantizec                    | j                   }d|v }|j                  }t        |t        j                        }|rMt        j                  ||j                  |j                  |j                  |j                  |j                        }|j                  \  }}t        j                  |||      }| j                  | j                  j                   z  | j"                  j                   z  j%                  |j&                        }	||	z   |_        |r|j(                  |_        |rC|sAt        j                  j+                  ||j                  |j                  |j                        }|S )Nbias
group_sizer   moder   )r   )r   r   r   r   r   mxr   scalesbiasesr   r   r   r   Linearr   lora_bTlora_aastypedtyper   from_linear)
selfr   r   r   r   is_quantizedr   r   fused_lineardeltas
             r   fusezLoRALinear.fuse"   s   !&"*<*<=]]!,,[[[[F #),,Zyy[tD**t{{}},=EEfllS$un &L
--99!![[	 : L r   r   r   r   c                 ^   t         |           t        j                  |||      | _        t        j
                  |      | _        || _        dt        j                  |      z  }t        j                  j                  | |||f      | _        t        j                  ||f      | _        y Nr    p   lowhighr   r   )super__init__r   r$   r   Dropoutr
   r   mathsqrtr!   randomuniformr'   zerosr%   )r+   r   r   r	   r
   r   r   	__class__s          r   r:   zLoRALinear.__init__C   s     	 ii
KdCzzG, 
 DIIj))ii''q/ ( 

 hha%56r   c                     | j                  |      }| j                  |      | j                  z  | j                  z  }|| j                  |z  j                  |j                        z   S N)r   r
   r'   r%   r   r(   r)   r+   xyzs       r   __call__zLoRALinear.__call___   sP    KKN\\!_t{{*dkk9DJJN**177333r                    4@FrJ   rK   rL   F)__name__
__module____qualname__staticmethodr   r$   intfloatr   boolr/   r:   rH   __classcell__rA   s   @r   r   r      s     			  	 *t J 77 7 	7
 7 7 784r   r   c                        e Zd Ze	 	 	 ddej
                  dededefd       Zdde	fdZ
	 	 	 	 dded	ed
edededede	f fdZddZ xZS )LoRASwitchLinearr   r	   r
   r   c                 r    t        | j                  | j                  | j                  |||      }| |_        |S )N)r   r   num_expertsr	   r
   r   )rY   r   r   r[   r   )r   r	   r
   r   r   s        r   r   zLoRASwitchLinear.from_basef   s@     $((****
 !r   r   c                    | j                   }d|v }|j                  }t        |t              }|rMt	        j
                  ||j                  |j                  |j                  |j                  |j                        }|j                  \  }}}t        ||||      }	| j                  | j                  z  }
| j                  j!                  |d|      }||
|z  j#                  |j$                        z   |	_        |r|j&                  |	_        |r4|s2|	j)                  |j                  |j                  |j                        }	|	S )Nr   r   r    )r   r   r   r   r!   r   r"   r#   r   r   r   r   r   r   r%   r'   reshaper(   r)   r   to_quantized)r+   r   r   r   r   r,   r[   r   r   r-   r%   r'   s               r   r/   zLoRASwitchLinear.fusex   s   !&*?@]]!,,[[[[F 06||,[*#J[tTdkk)$$["jA$'?'?'MM &L
'44!,,6;;V[[ 5 L r   r   r   r[   r   c                 ^   t         |           t        ||||      | _        t	        j
                  |      | _        || _        dt        j                  |      z  }t        j                  j                  | ||||f      | _        t        j                  |||f      | _        || _        y r1   )r9   r:   r   r   r   r;   r
   r   r<   r=   r!   r>   r?   r'   r@   r%   r[   )	r+   r   r   r[   r	   r
   r   r   rA   s	           r   r:   zLoRASwitchLinear.__init__   s     	 #:{KdSzzG, 
 DIIj))ii'':. ( 

 hhk;%BC&r   c                 j   | j                  |||      }t        j                  | j                  |      | j                  j                  dd      ||      }t        j                  || j                  j                  dd      ||      }|| j                  |z  j                  |j                        z   S )N)sorted_indicesr]   )rhs_indicesrb   )
r   r!   	gather_mmr
   r'   swapaxesr%   r   r(   r)   )r+   rE   indicesrb   rF   rG   s         r   rH   zLoRASwitchLinear.__call__   s    KK7>KBLLLLOKK  R()	
 LLKK  R()	
 DJJN**177333r   rI   rM   rN   )rO   rP   rQ   rR   r   ModulerS   rT   r   rU   r/   r:   rH   rV   rW   s   @r   rY   rY   e   s     			  	 "t H '' ' 	'
 ' ' ' '<4r   rY   c                        e Zd Ze	 	 	 ddej
                  dededefd       Zdde	fdZ
	 	 	 dded	edededef
 fd
Zd Zd Z xZS )LoRAEmbedding	embeddingr	   r
   r   c                     | j                   j                  \  }}t        | t        j                        r|dz  | j
                  z  }t        |||||      }| |_        |S )Nr   )num_embeddingsdimsr	   r
   r   )r   r   r   r   QuantizedEmbeddingr   rj   rk   )rk   r	   r
   r   rm   rn   lora_embeddings          r   r   zLoRAEmbedding.from_base   se      )//55i!6!67"9	.D&)
 $- r   r   c                 r   | j                   }|j                  }t        |t        j                        }|rMt        j                  ||j                  |j                  |j                  |j                  |j                        }|j                  \  }}t        j                  ||      }| j                  | j                  z  }| j                   }	|||	z  j#                  |j$                        z   |_        |rC|sAt        j                  j'                  ||j                  |j                  |j                        }|S )Nr   )rk   r   r   r   ro   r!   r   r"   r#   r   r   r   r   	Embeddingr   r'   r%   r(   r)   from_embedding)
r+   r   rk   r   r,   rm   rn   fused_embeddingr'   r%   s
             r   r/   zLoRAEmbedding.fuse   s   NN	!!!)R-B-BC]]    $//^^^^F  &||,,~t<dkk)!'6F?*B*B6<<*P!P
 33BB$//^^^^	 C O r   rm   rn   c                 Z   t         |           t        j                  ||      | _        t        j
                  |      | _        || _        dt        j                  |      z  }t        j                  j                  | |||f      | _        t        j                  ||f      | _        y )Nr2   r4   r5   r8   )r9   r:   r   rr   rk   r;   r
   r   r<   r=   r!   r>   r?   r'   r@   r%   )r+   rm   rn   r	   r
   r   rA   s         r   r:   zLoRAEmbedding.__init__   s     	 nd;zzG, 
 DIIn--ii''!1% ( 

 hhaY/r   c                     | j                  |      }| j                  | j                  |   | j                  z        }|| j                  |z  j                  |j                        z   }|S rC   )rk   r
   r'   r%   r   r(   r)   )r+   rE   rF   rG   outs        r   rH   zLoRAEmbedding.__call__  sU    NN1LLQ$++564::>))!''22
r   c                    | j                   j                  |      }| j                  |      | j                  j                  z  | j
                  j                  z  }|| j                  |z  j                  |j                        z   S rC   )	rk   	as_linearr
   r%   r&   r'   r   r(   r)   rD   s       r   ry   zLoRAEmbedding.as_linear  s_    NN$$Q'\\!_t{{}},=DJJN**177333r   rI   rM   )rO   rP   rQ   rR   r   rr   rS   rT   r   rU   r/   r:   rH   ry   rV   rW   s   @r   rj   rj      s     	<<  	 &t F 00 0 	0
 0 044r   rj   )r<   mlx.corecorer!   mlx.nnr   models.switch_layersr   r   rh   r   rY   rj    r   r   <module>r      sK       FW4 W4t^4ryy ^4BW4BII W4r   