
    (HJj                     l   d dl Z d dlmZ d dlmZ d dlmZ ddlm	Z	 ddl
mZmZmZ e G d de             Z G d d	ej                        Z G d
 dej                        Z G d dej                        Z G d dej                        Z G d dej                        Z G d dej                        Zy)    N)	dataclass   )swiglu)BaseModelArgscreate_attention_maskscaled_dot_product_attentionc                       e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed	<   eed
<   eed<   dZeed<   dZ	eed<   y)	ModelArgs
model_type
vocab_sizehidden_sizenum_attention_headsnum_hidden_layersnum_key_value_headsintermediate_size
rope_thetause_qkv_biaspartial_rotary_factorlayer_norm_epsFuse_parallel_residualqk_layernormN)
__name__
__module____qualname__str__annotations__intfloatboolr   r        `/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx_lm/models/stablelm.pyr
   r
      sW    OO  "'4'L$r!   r
   c                   $     e Zd Z fdZd Z xZS )LayerNormPerHeadc                     t         |           t        |      D cg c]  }t        j                  ||d       c}| _        || _        y c c}w )NF)epsbias)super__init__rangenn	LayerNormnormsr&   )selfhead_dim	num_headsr&   _	__class__s        r"   r)   zLayerNormPerHead.__init__    sJ    AFyAQ
AQABLLs7AQ

 
s   Ac                     t        j                  | j                  D cg c]  }|j                   c}      }|t         j                  j                  |d d | j                        z  S c c}w N)mxstackr-   weightfast
layer_normr&   )r.   xnws       r"   __call__zLayerNormPerHead.__call__'   sP    HH

3
1ahh
34277%%atTXX>>> 4s   A')r   r   r   r)   r=   __classcell__r2   s   @r"   r$   r$      s    ?r!   r$   c                   ,     e Zd Zdef fdZddZ xZS )	Attentionconfigc                 Z   t         |           |j                  | _        |j                  | _        | j                  | j                  z  | _        |j                  | _        |j                  | _        |j                  | _        | j
                  | j                  z  | j                  k7  r&t        d| j                   d| j                   d      t        j                  | j                  | j                  | j
                  z  |j                        | _        t        j                  | j                  | j                  | j
                  z  |j                        | _        t        j                  | j                  | j                  | j
                  z  |j                        | _        t        j                  | j                  | j
                  z  | j                  d      | _        t        j"                  t%        | j                  | j
                  z        d| j                        | _        |j(                  | _        | j(                  rct+        | j
                  | j                  |j,                        | _        t+        | j
                  | j                  |j,                        | _        y y )Nz?hidden_size must be divisible by num_heads (got `hidden_size`: z and `num_heads`: z).r'   F)traditionalbaser&   )r(   r)   r   r   r0   r/   r   r   r   
ValueErrorr+   Linearr   q_projk_projv_projo_projRoPEr   roper   r$   r   q_layernormk_layernormr.   rB   r2   s     r"   r)   zAttention.__init__-   s   !--33((DNN:#)#=#=  ++%+%A%A"MMDNN*t/?/??QRVRbRbQc$T^^$4B8 
 iidnnt}}<6CVCV
 ii$$t}}4$$

 ii$$t}}4$$

 iiNNT]]*D,<,<5
 GG**T]]:;
	 #///t~~63H3H D  0t77V=R=R D	 r!   c                 z   | j                  |      | j                  |      | j                  |      }}}|j                  \  }}}	|j	                  ||| j
                  d      }|j	                  ||| j                  d      }| j                  r"| j                  |      }| j                  |      }|j                  dddd      }|j                  dddd      }|j	                  ||| j                  d      j                  dddd      }|P| j                  ||j                        }| j                  ||j                        }|j                  ||      \  }}n"| j                  |      }| j                  |      }|j                  t        j                         }|j                  t        j                         }t#        j$                  d|j                  d   z        }
t'        |||||
|      j                  |j(                        }|j                  dddd      j	                  ||d      }| j+                  |      S )Nr      r      )offset)cachescalemask)rJ   rK   rL   shapereshaper0   r   r   rP   rQ   	transposerO   rW   update_and_fetchastyper5   float32mathsqrtr   dtyperM   )r.   r:   rZ   rX   querieskeysvaluesBLDrY   outputs               r"   r=   zAttention.__call__]   s    $AAAv --1a//!Q;||Aq$":":B?&&w/G##D)D##Aq!Q/~~aAq)1d&>&>CMMq!Q

 iii=G99T%,,97D 11$?LD&ii(G99T?D..,{{2::& 		!gmmB//0-T6e$

&
 	 !!!Q1-55aB?{{6""r!   )NNr   r   r   r
   r)   r=   r>   r?   s   @r"   rA   rA   ,   s    .y .`##r!   rA   c                   >     e Zd Z fdZdej
                  fdZ xZS )MLPc                     t         |           t        j                  ||d      | _        t        j                  ||d      | _        t        j                  ||d      | _        y NFrD   )r(   r)   r+   rI   	gate_proj	down_projup_proj)r.   dim
hidden_dimr2   s      r"   r)   zMLP.__init__   sJ    3
?:s?yyju=r!   returnc                 t    | j                  t        | j                  |      | j                  |                  S r4   )rq   r   rp   rr   )r.   r:   s     r"   r=   zMLP.__call__   s)    ~~fT^^A%6QHIIr!   )r   r   r   r)   r5   arrayr=   r>   r?   s   @r"   rm   rm      s    >JRXX Jr!   rm   c                   *     e Zd Zdef fdZd Z xZS )DecoderLayerrB   c                    t         |           t        |      | _        t	        |j
                  |j                        | _        t        j                  |j
                  |j                        | _        |j                  | _        | j                  s1t        j                  |j
                  |j                        | _        y y )N)rB   rG   )r(   r)   rA   	self_attnrm   r   r   mlpr+   r,   r   input_layernormr   post_attention_layernormrR   s     r"   r)   zDecoderLayer.__init__   s    "&1v))6+C+CD!||%% 
 &,%A%A")),.LL""))-D) *r!   c                     | j                  |      }| j                  |||      }| j                  r||z   | j                  |      z   }|S ||z   }| j                  | j	                  |            }||z   }|S r4   )r}   r{   r   r|   r~   )r.   r:   rZ   rX   hrouts          r"   r=   zDecoderLayer.__call__   s|      #NN1dE*%%a%$((1+%C
 
 AA66q9:Aa%C
r!   rk   r?   s   @r"   ry   ry      s    y 
r!   ry   c                   *     e Zd Zdef fdZd Z xZS )StableLMrB   c                 L   t         |           t        j                  |j                  |j
                        | _        t        |j                        D cg c]  }t        |       c}| _
        t        j                  |j
                  |j                        | _        y c c}w )NrG   )r(   r)   r+   	Embeddingr   r   embed_tokensr*   r   ry   layersr,   r   norm)r.   rB   ir2   s      r"   r)   zStableLM.__init__   sy    LL):):F<N<NO5:6;S;S5TU5T|F+5TULL!3!39N9NO	 Vs   B!c                     | j                  |      }|d gt        | j                        z  }t        ||d         }t	        | j                  |      D ]  \  }} ||||      } | j                  |      S )Nr   )rX   )r   lenr   r   zipr   )r.   r:   rX   rZ   layercs         r"   r=   zStableLM.__call__   ss    a =FS--E$Qa1DKK/HE1aQ'A 0 yy|r!   rk   r?   s   @r"   r   r      s    Py P
r!   r   c                   p     e Zd Zdef fdZ	 ddej                  dej                  fdZed        Z	 xZ
S )ModelrB   c                     t         |           |j                  | _        t        |      | _        t        j                  |j                  |j                  d      | _	        || _
        y ro   )r(   r)   r   r   modelr+   rI   r   r   lm_headargsrR   s     r"   r)   zModel.__init__   sN     ++f%
yy!3!3V5F5FUS	r!   r:   ru   c                 H    | j                  ||      }| j                  |      S r4   )r   r   )r.   r:   rX   ys       r"   r=   zModel.__call__   s"    
 JJq% ||Ar!   c                 .    | j                   j                  S r4   )r   r   )r.   s    r"   r   zModel.layers   s    zz   r!   r4   )r   r   r   r
   r)   r5   rw   r=   propertyr   r>   r?   s   @r"   r   r      sG    y  88 
	 ! !r!   r   )ra   dataclassesr   mlx.corecorer5   mlx.nnr+   activationsr   rF   r   r   r   r
   Moduler$   rA   rm   ry   r   r   r    r!   r"   <module>r      s     !    T T    ?ryy ?T#		 T#nJ")) J299 :ryy (!BII !r!   