
    (HJjJ              	       n   d dl Z d dlmZ d dlmZ d dlmZmZmZm	Z	 d dl
mZ d dlmZ d dlmZmZmZmZ ddlmZ 	 d6dej,                  d	e	e   d
ee   fdZi fdedee   deeef   fdZd7dZdee   d
efdZdee   ded
ee   fdZdededede	e   fdZ G d d      Z  G d de       Z! G d de       Z" G d d e       Z# G d! d"e       Z$ G d# d$e       Z% G d% d&e       Z& G d' d(e       Z'd) Z( G d* d+e       Z) G d, d-e       Z* G d. d/      Z+e G d0 d1             Z, G d2 d3      Z- G d4 d5      Z.y)8    N)deque)	dataclass)AnyDictListOptional)tree_flattentree_maptree_reducetree_unflatten   create_causal_maskmodelmax_kv_sizereturnc                    t        | d      r| j                         S t        | j                        }|$t	        |      D cg c]  }t        |d       c}S t	        |      D cg c]  }t                c}S c c}w c c}w )a  
    Construct the model's cache for use in generation.

    This function will defer the cache construction to the model if it has a
    ``make_cache`` method, otherwise it will make a default KV cache.

    Args:
        model (nn.Module): The language model.
        max_kv_size (Optional[int]): If provided and the model does not have a
            ``make_cache`` method, a ``RotatingKVCache`` is used with a maximum
            size of ``max_kv_size``
    
make_cache   )max_sizekeep)hasattrr   lenlayersrangeRotatingKVCacheKVCache)r   r   
num_layers_s       ]/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx_lm/models/cache.pymake_prompt_cacher!      s      ul#!!U\\"JCHCT
CTaO[q9CT
 	
 $)#45#4a	#455	
 6s   A9%A>	file_namecachemetadatac                 R   |D cg c]  }|j                    }}|D cg c]  }|j                   }}t        t        |            }|D cg c]  }t	        |      j
                   }}|||g}t        t        |            }t        j                  | ||       yc c}w c c}w c c}w )a  
    Save a pre-computed prompt cache to a file.

    Args:
        file_name (str): The ``.safetensors`` file name.
        cache (List[Any]): The model state.
        metadata (Dict[str, str]): Optional metadata to save along with model
            state.
    N)state
meta_statedictr	   type__name__mxsave_safetensors)r"   r#   r$   c
cache_data
cache_infocache_classescache_metadatas           r    save_prompt_cacher2   +   s     $))5a!''5J)(-.1!,,J.l:./J/45u!T!W%%uM5 (M:N,~67N	:~> *.5s   BBB$c           	      b   t        j                  | d      \  }}t        t        |j	                                     }t        t        |j	                                     }|\  }}}t        |||      D 	cg c]#  \  }}}	t               |   j                  ||	      % }
}}}	|r|
|fS |
S c c}	}}w )aR  
    Load a prompt cache from a file.

    Args:
        file_name (str): The ``.safetensors`` file name.
        return_metadata (bool): Whether or not to return metadata.
            Default: ``False``.

    Returns:
        List[Any] or Tuple[List[Any], Dict[str, str]]: The prompt cache and
            the metadata if requested.
    T)return_metadata)r+   loadr   listitemszipglobals
from_state)r"   r4   arraysr1   infor$   classesr-   r&   r'   r#   s              r    load_prompt_cacher>   >   s      WWYEFND01F#D)=)=)?$@AN,D(G %($>$> Auj 		!z2$> 
  hLs   6(B*c                 &    t        d | D              S )z0
    Check if model's cache can be trimmed.
    c              3   <   K   | ]  }|j                           y wNis_trimmable.0r-   s     r    	<genexpr>z(can_trim_prompt_cache.<locals>.<genexpr>\   s     /Aq~~   )allr#   s    r    can_trim_prompt_cacherJ   X   s     ////    
num_tokensc                     t        |       rt        |       dk(  ry| D cg c]  }|j                  |       c}d   S c c}w )aj  
    Trim the model's cache by the given number of tokens.

    This function will trim the cache if possible (in-place) and return the
    number of tokens that were trimmed.

    Args:
        cache (List[Any]): The model's cache.
        num_tokens (int): The number of tokens to trim.

    Returns:
        (int): The number of tokens that were trimmed.
    r   )rJ   r   trim)r#   rL   r-   s      r    trim_prompt_cacherO   _   s?     !'3u:?(-.1AFF:.q11.s   =Noffsetreturn_arraywindow_sizec                 P    |t        | ||      S | dk(  ry |rt        | ||      S y)NrS   r   causalr   )rP   rQ   rR   rS   s       r    create_attention_maskrW   r   s8     !!VEE	
a	!!VEErK   c                       e Zd Zed        Zej
                  d        Zed        Zej
                  d        Zd Zd Zed        Z	d Z
ed	        Zy
)
_BaseCachec                     g S rA    selfs    r    r&   z_BaseCache.state   s    	rK   c                 $    ||rt        d      y y )Nz,This cache has no state but a state was set.
ValueErrorr]   vs     r    r&   z_BaseCache.state   s    =QKLL =rK   c                      y)N r[   r\   s    r    r'   z_BaseCache.meta_state   s    rK   c                 $    ||rt        d      y y )Nz6This cache has no meta_state but a meta_state was set.r_   ra   s     r    r'   z_BaseCache.meta_state   s    =QUVV =rK   c                      yNFr[   r\   s    r    rC   z_BaseCache.is_trimmable   s    rK   c                      y)z
        Return the size (i.e. sequence length) of the cache.

        Not every cache is required to implement this, in which case the size
        will always be 0 (though the cache may not be empty).
        r   r[   r\   s    r    sizez_BaseCache.size   s     rK   c                     t        d      )z&Return the size of this cache in bytesz%Cache sub-class must implement nbytesNotImplementedErrorr\   s    r    nbytesz_BaseCache.nbytes   s     ""IJJrK   c                     t        d      )z6
        Return if the cache is empty or not.
        z$Cache sub-class must implement this.rk   r\   s    r    emptyz_BaseCache.empty   s     ""HIIrK   c                 D    | j                  |       }||_        ||_        |S rA   )__new__r&   r'   )clsr&   r'   objs       r    r:   z_BaseCache.from_state   s%     kk#	#
rK   N)r*   
__module____qualname__propertyr&   setterr'   rC   ri   rm   ro   classmethodr:   r[   rK   r    rY   rY      s      \\M M   W W K KJ  rK   rY   c                   x    e Zd ZdZd Zd Zed        Zej                  d        Zd Z	d Z
d Zd	 Zed
        Zy)ConcatenateKVCachea  ConcatenateKVCache the simplest KV cache implementation.

    Can be used as a mock KV cache or when large blocks are being processed at
    a time in which case KVCache isn't necessarily faster. Consider using the
    KVCache with a larger step size before using this cache.
    c                 .    d | _         d | _        d| _        y Nr   keysvaluesrQ   r\   s    r    __init__zConcatenateKVCache.__init__       	rK   c                 D   | j                   || _         || _        nPt        j                  | j                   |gd      | _         t        j                  | j                  |gd      | _        | j                   j                  d   | _        | j                   | j                  fS )Naxis)r~   r   r+   concatenateshaperQ   r]   r~   r   s      r    update_and_fetchz#ConcatenateKVCache.update_and_fetch   sw    99DI DK		4'8rBDI..$++v)>RHDKiioob)yy$++%%rK   c                 2    | j                   | j                  fS rA   r~   r   r\   s    r    r&   zConcatenateKVCache.state   s    yy$++%%rK   c                 ^    |\  | _         | _        | j                   j                  d   | _        y )Nr   r~   r   r   rQ   ra   s     r    r&   zConcatenateKVCache.state   s$    !"	4;iioob)rK   c                      yNTr[   r\   s    r    rC   zConcatenateKVCache.is_trimmable       rK   c                 \    t        | j                  |      }| xj                  |z  c_        |S rA   minrQ   r]   ns     r    rN   zConcatenateKVCache.trim   %    QqrK   c                 0    t        |d| j                  i|S NrQ   rW   rQ   r]   argskwargss      r    	make_maskzConcatenateKVCache.make_mask       $dI4;;I&IIrK   c                     | j                   d u S rA   r~   r\   s    r    ro   zConcatenateKVCache.empty       yyD  rK   c                 v    | j                   y| j                   j                  | j                  j                  z   S r|   r~   rm   r   r\   s    r    rm   zConcatenateKVCache.nbytes   .    99yy$++"4"444rK   N)r*   rt   ru   __doc__r   r   rv   r&   rw   rC   rN   r   ro   rm   r[   rK   r    rz   rz      sf    
	& & & \\* *
J! 5 5rK   rz   c                       e Zd ZdZddedefdZd Zed        Zej                  d        Zed        Z
e
j                  d	        Z
d
 Zd Zd Zd Zed        Zy)QuantizedKVCache   
group_sizebitsc                 J    d | _         d | _        d| _        || _        || _        y r|   )r~   r   rQ   r   r   r]   r   r   s      r    r   zQuantizedKVCache.__init__   s%    	$	rK   c                     j                   \  }}}}|j                   d   } j                   j                  #|z    j                  d   j                   d   kD  rdt        j                  j
                  z   j                  z   j                  |z   dz
   j                  z   j                  z  }|||f fd}	fd}
 j                  t j                  z  dk7  r2t        fd j                   j                  f      \   _         _	        t        |
 j                   j                  f      \   _         _	        n |	|       |	|      c _         _	         xj                  |z  c_        t        j                   j                   j                  	      t        j                  | j                   j                  	      }t        t         j                              D ]N  }|    j                  |   d
 j                  d d f<   ||    j                  |   d
 j                  d d f<   P t         fd j                   j                  f      S )Nr   r      r   c                 (   t        j                  g | z  t         j                        t        j                  g | j                  z  j                        t        j                  g | j                  z  j                        fS Ndtype)r+   zerosuint32r   r   )dim
el_per_intr~   r]   r   s    r    
init_quantz5QuantizedKVCache.update_and_fetch.<locals>.init_quant   s{    HH8u8cZ&78		JHH=u=cT__&<=TZZPHH=u=cT__&<=TZZP rK   c                     t        j                  g | j                  d   | j                        }t        j                  | |gd      S )Nr   r   r   r   )r+   r   r   r   r   )xnew_xr   s     r    expand_quantz7QuantizedKVCache.update_and_fetch.<locals>.expand_quant  s?    !65!6!''"+!6aggF~~q%jr::rK   c                     | dd d d f   S N.r[   )r   prevs    r    <lambda>z3QuantizedKVCache.update_and_fetch.<locals>.<lambda>
  s    !C$M"2rK   r   r   .c                 0    | dd j                   d d f   S r   rQ   r   r]   s    r    r   z3QuantizedKVCache.update_and_fetch.<locals>.<lambda>  s    !C4;;$9":rK   )r   rQ   r~   r+   r   ri   r   stepr
   r   quantizer   r   r   )r]   r~   r   B
n_kv_heads	num_steps
k_head_dim
v_head_dim	new_stepsr   r   ir   r   r   s   ``          @@@r    r   z!QuantizedKVCache.update_and_fetch   s   /3zz,:y*\\"%
{{99	!1TYYq\5G5G5K KRYY^^+tyy8JY.2tyy@499LI
I.E; yy$$))#q(-52TYY4L.*DIt{ *2 499dkk":*&	4; *4J)?JAW&	4;y {{4DOO$))LVdiiPs499~&A7;AwDIIaLdT[[0!349?DKKN3t{{ 2A56 ' :TYY<TUUrK   c                       j                    j                  d   j                  d   k(  r j                   j                  fS t	         fd j                   j                  f      S )Nr      c                 0    | dd j                   d d f   S r   r   r   s    r    r   z(QuantizedKVCache.state.<locals>.<lambda>#  s    !C4;;12rK   )rQ   r~   r   r   r
   r\   s   `r    r&   zQuantizedKVCache.state  sT    ;;$))A,,,Q//99dkk))2TYY4L rK   c                 "    |\  | _         | _        y rA   r   ra   s     r    r&   zQuantizedKVCache.state&      !"	4;rK   c                 v    t        t        t        | j                  | j                  | j
                  f            S rA   )tuplemapstrrQ   r   r   r\   s    r    r'   zQuantizedKVCache.meta_state*  s'    St{{DOOTYYGHIIrK   c                 J    t        t        |      \  | _        | _        | _        y rA   )r   intrQ   r   r   ra   s     r    r'   zQuantizedKVCache.meta_state.  s    25c1+/T_dirK   c                      yr   r[   r\   s    r    rC   zQuantizedKVCache.is_trimmable2  r   rK   c                 \    t        | j                  |      }| xj                  |z  c_        |S rA   r   r   s     r    rN   zQuantizedKVCache.trim5  r   rK   c                 0    t        |d| j                  i|S r   r   r   s      r    r   zQuantizedKVCache.make_mask:  r   rK   c                     | j                   d u S rA   r   r\   s    r    ro   zQuantizedKVCache.empty=  r   rK   c                 J    t        d | j                  | j                  fd      S )Nc                      | |j                   z   S rA   rm   )ar   s     r    r   z)QuantizedKVCache.nbytes.<locals>.<lambda>B  s    AHHrK   r   )r   r~   r   r\   s    r    rm   zQuantizedKVCache.nbytes@  s     4tyy$++6NPQRRrK   N)@   r   )r*   rt   ru   r   r   r   r   rv   r&   rw   r'   rC   rN   r   ro   rm   r[   rK   r    r   r      s    D3 3 )VV   \\# # J J > >
J! S SrK   r   c                       e Zd ZdZd Zd Zd Zed        Zej                  d        Zd Z
d Zdd	ed
edefdZd Zed        Zd Zed        Zy)r   r   c                 .    d | _         d | _        d| _        y r|   r}   r\   s    r    r   zKVCache.__init__H  r   rK   c                    | j                   }| j                  -||j                  d   z   | j                  j                  d   kD  rf|j                  \  }}}}|j                  d   }| j                  |j                  d   z   dz
  | j                  z  }	|||	| j                  z  |f}
|||	| j                  z  |f}t	        j
                  |
|j                        }t	        j
                  ||j                        }| j                  || j                  z  dk7  r6| j                  dd |d d f   | _        | j                  dd |d d f   | _        t	        j                  | j                  |gd      | _        t	        j                  | j                  |gd      | _        n||c| _        | _        | xj                   |j                  d   z  c_         || j                  d|| j                   d d f<   || j                  d|| j                   d d f<   | j                  dd | j                   d d f   | j                  dd | j                   d d f   fS Nr      r   r   .r   )	rQ   r~   r   r   r+   r   r   r   r   r]   r~   r   r   r   r   r   r   r   n_stepsk_shapev_shapenew_knew_vs                 r    r   zKVCache.update_and_fetchM  s   {{99

1!59K K+/::(Az1jaJyy4::a=014BG*g		&9:FG*g		&9:FGHHWdjj1EHHWfll3Eyy$$))#q( $		#uua- 8DI"&++c5D5!m"<DKNNDIIu+=AF	 nndkk5-AJ).&	4;tzz!}$04		#tdkk)1,-28C+Q./yymmQ./S-DKK-QR=R1SSSrK   c                     | j                   S rA   r   r\   s    r    ri   zKVCache.sizee  s    {{rK   c                     | j                   | j                  j                  d   k(  r| j                  | j                  fS | j                  dd | j                   d d f   | j                  dd | j                   d d f   fS Nr   .rQ   r~   r   r   r\   s    r    r&   zKVCache.stateh  p    ;;$))//!,,99dkk)) 		#}}a/0C4;;12 rK   c                 ^    |\  | _         | _        | j                   j                  d   | _        y Nr   r   ra   s     r    r&   zKVCache.stater  $    !"	4;iiooa(rK   c                      yr   r[   r\   s    r    rC   zKVCache.is_trimmablew  r   rK   c                 \    t        | j                  |      }| xj                  |z  c_        |S rA   r   r   s     r    rN   zKVCache.trimz  r   rK   r   r   r   c                     t        ||      }| j                  |_        | j                  Nt        j                  | j                  ||      |_        t        j                  | j
                  ||      |_        |S )Nr   )r   rQ   r~   r+   r   r   )r]   r   r   quant_caches       r    to_quantizedzKVCache.to_quantized  sb    &*4H![[99 !{{499RVWK!#
"K rK   c                 0    t        |d| j                  i|S r   r   r   s      r    r   zKVCache.make_mask  r   rK   c                 ,    t         j                  |      S rA   )BatchKVCachemerger   cachess     r    r   zKVCache.merge  s    !!&))rK   c                     | j                   d u S rA   r   r\   s    r    ro   zKVCache.empty  r   rK   c                 v    | j                   y| j                   j                  | j                  j                  z   S r|   r   r\   s    r    rm   zKVCache.nbytes  r   rK   Nr   r   )r*   rt   ru   r   r   r   ri   rv   r&   rw   rC   rN   r   r   r   r   rx   r   ro   rm   r[   rK   r    r   r   E  s    D
T0   \\) )
s s CS J * *! 5 5rK   r   c                      e Zd ZdZddZddZd Zd Zd Zd Z	d	 Z
ed
        Zej                  d        Zed        Zej                  d        Zd Zd ZddededefdZ	 ddedee   defdZed        Zd Zed        Zy)r   r   c                 X    || _         d | _        d | _        d| _        || _        d| _        y r|   )r   r~   r   rQ   r   _idx)r]   r   r   s      r    r   zRotatingKVCache.__init__  s,    		 	rK   Nc                     g }|dkD  r0|dd | j                   d d f   |d|| j                   z   d d d f   g}n|g}||j                  |       t        j                  |d      S Nr   .r   r   )r   appendr+   r   )r]   	trim_sizerb   r  to_cats        r    _trimzRotatingKVCache._trim  sn    q=[tyy[!+,aY5J5La0O.PQFSFMM&!~~f1--rK   c           	      \   | j                   |j                  d   k(  r|S | j                   | j                  k  r`t        j                  |dd| j
                  ddf   |d| j                   dddf   |d| j
                  | j                   ddf   gd      S |dd| j                   ddf   S )zY
        Rearrange the cache into temporal order, slicing off the end if unused.
        r   .Nr   )r  r   rQ   r+   r   r   ra   s     r    _temporal_orderzRotatingKVCache._temporal_order  s     99
"HYY$>>c;TYY;)*c499;)*c499tyy0!34
   S+DII+q())rK   c                 d   | j                   || _         || _        n| j                  | j                         | _         | j                  | j                        | _        | j                   j                  d   | _        | j                  | j
                  z
  dz   }| j                  || j                   |      | _         | j                  || j                  |      | _        | xj                  |j                  d   z  c_        | j                   j                  d   | _        | j                   | j                  fS Nr   r   )r~   r   r  r   r  r   r  rQ   )r]   r~   r   r	  s       r    _update_concatzRotatingKVCache._update_concat  s    99DI DK ,,TYY7DI..t{{;DK		*DI 		DMM1A5I

9dii>DI**YVDDKtzz!}$IIOOA&	yy$++%%rK   c                    |j                   \  }}}}| j                  }| j                  C|| j                  j                   d   k\  r| j                  j                   d   | j                  k  r|j                   d   }t	        | j
                  | j                  |z
        }	|||	|f}
|||	|f}t        j                  |
|j                        }t        j                  ||j                        }| j                  Qt        j                  | j                  |gd      | _        t        j                  | j                  |gd      | _
        n||c| _        | _
        || _        | j                  j                   d   | j                  z
  }|dkD  rS| j                  || j                        | _        | j                  || j                        | _
        | j                  | _        | j                  | j                  k(  r| j                  | _        || j                  d| j                  | j                  |z   d d f<   || j                  d| j                  | j                  |z   d d f<   | xj                  |z  c_        | xj                  |z  c_        | j                  | j                  k  r@| j                  dd | j                  d d f   | j                  dd | j                  d d f   fS | j                  | j                  fS )Nr   r   r   r   .)r   rQ   r~   r   r   r   r+   r   r   r   r   r  r  r   r]   r~   r   r   r   Sr   r   r   new_sizer   r   r   r   r	  s                  r    _update_in_placez RotatingKVCache._update_in_place  sY    (,zz$:q*{{99DIIOOA&&499??1+=+MaJ499dmmd&:;H*h
;G*h
;GHHWdjj1EHHWfll3Eyy$NNDIIu+=AF	 nndkk5-AJ).&	4;DI IIOOA&6	q=

9dii8DI**Y<DKDI 99%		DI 8<		#tyy499q=0!349?CTYY]2A56q		Q	 ;;&99S-DKK-23T[[mmUVAV5WWWyy$++%%rK   c                 n    |j                   d   dk(  r| j                  ||      S | j                  ||      S r  r   r  r  r   s      r    r   z RotatingKVCache.update_and_fetch   8    ::a=A((v66""400rK   c                 B    t        | j                  | j                        S rA   )r   rQ   r   r\   s    r    ri   zRotatingKVCache.size  s    4;;..rK   c                     | j                   | j                  j                  d   k  r@| j                  dd | j                   d d f   | j                  dd | j                   d d f   fS | j                  | j                  fS r   r   r\   s    r    r&   zRotatingKVCache.state  sj    ;;++99S-DKK-23T[[mmUVAV5WWW99dkk))rK   c                 "    |\  | _         | _        y rA   r   ra   s     r    r&   zRotatingKVCache.state  r   rK   c           	          t        t        t        | j                  | j                  | j
                  | j                  f            S rA   )r   r   r   r   r   rQ   r  r\   s    r    r'   zRotatingKVCache.meta_state  s-    Styy$--diiPQRRrK   c                 V    t        t        |      \  | _        | _        | _        | _        y rA   )r   r   r   r   rQ   r  ra   s     r    r'   zRotatingKVCache.meta_state  s#    ;><
8	4=$+tyrK   c                 4    | j                   | j                  k  S rA   )rQ   r   r\   s    r    rC   zRotatingKVCache.is_trimmable  s    {{T]]**rK   c                     t        | j                  |      }| xj                  |z  c_        | xj                  |z  c_        |S rA   )r   rQ   r  r   s     r    rN   zRotatingKVCache.trim!  s2    Qq		Q	rK   r   r   r   c                     t        d      )Nz RotatingKVCache Quantization NYIrk   r   s      r    r   zRotatingKVCache.to_quantized'  s    !"DEErK   rP   rS   rR   c                    |dkD  rL|xs | j                   }t        | j                   dz
  | j                        }||z   |kD  s|rt        |||      S y|y | j                  |k\  r| j                   |kD  r| j                  }|| j                   k\  rd}| j                  | j                   k  r| j                  dz   }n| j                   }t        j                  |      ||z
  k\  }t        j                  ||dz         }|S y y )Nr   rU   rV   r   )shift)r   r   rQ   r   r  r+   arangeroll)r]   rP   rS   rR   rQ   idx	mask_sizemasks           r    r   zRotatingKVCache.make_mask*  s     q5%6K*DKK8FzK'<)!VMM"{{k)dmmk.Iii$--'C;;. $aI $Iyy+	K0GHwwt373 /J)rK   c                 ,    t         j                  |      S rA   )BatchRotatingKVCacher   r   s     r    r   zRotatingKVCache.mergeD  s    #))&11rK   c                     | j                   d u S rA   r   r\   s    r    ro   zRotatingKVCache.emptyH  r   rK   c                 v    | j                   y| j                   j                  | j                  j                  z   S r|   r   r\   s    r    rm   zRotatingKVCache.nbytesK  r   rK   )r   rA   r  rg   )r*   rt   ru   r   r   r  r  r  r  r   ri   rv   r&   rw   r'   rC   rN   r   r   r   r   boolr   rx   r   ro   rm   r[   rK   r    r   r     s   D.*$&()&V1
/ * * \\# # S S 
 
+Fs Fs FCS F OT#+C=GK4 2 2! 5 5rK   r   c                        e Zd Z fdZddeee      fdZed        Z	d Z
d Zed        Zej                  d        Zd	 Zd
 Zd ZddZd Zd ZdefdZed        Zd Zed        Z xZS )ArraysCachec                 B    t         |   |       }d |_        d |_        |S rA   )superrq   left_paddinglengths)rr   r   r   instance	__class__s       r    rq   zArraysCache.__new__S  s&    7?3' $rK   r1  c                 T    d g|z  | _         |rt        j                  |      | _        y y rA   )r#   r+   arrayr1  )r]   ri   r1  s      r    r   zArraysCache.__init__Y  s(    Vd]
 " 6D rK   c                     | j                   D ]  }||j                  d   c S  | j                  | j                  j                  S | j                  | j                  j                  S yNr   r   )r#   r   r1  ri   r2  r]   r-   s     r    
batch_sizezArraysCache.batch_size^  s`    A}wwqz!  ($$)))\\%<<$$$rK   c                 "    || j                   |<   y rA   rI   )r]   r%  values      r    __setitem__zArraysCache.__setitem__j  s    

3rK   c                      | j                   |   S rA   rI   r]   r%  s     r    __getitem__zArraysCache.__getitem__m  s    zz#rK   c                     | j                   S rA   rI   r\   s    r    r&   zArraysCache.statep  s    zzrK   c                     || _         y rA   rI   ra   s     r    r&   zArraysCache.statet  s	    
rK   c                     | j                   D cg c]  }|||   nd c}| _         | j                  | j                  |   | _        | j                  | j                  |   | _        yyc c}w N
        In-place filter to keep just the given indices in the cache.
        N)r#   r1  r2  r]   batch_indicesr-   s      r    filterzArraysCache.filterx  sr     LP::V:a!-a&TA:V
( $ 1 1- @D<<#<<6DL $ Ws   A)c                 V   | j                   |j                   fd}t        | j                  |j                        D cg c]  \  }} |||       c}}| _         || j                  |j                        | _         || j                  |j                        | _        yc c}}w )B
        In-place extend this cache with the other cache.
        c                 (   d x}}| | j                   }| j                  }||j                   }|j                  }|y | t        j                  f|dd  z   |      } |t        j                  f|dd  z   |      }t        j                  | |g      S Nr   r   )r   r   r+   r   r   )r   br   r   a_batchb_batchs       r    catzArraysCache.extend.<locals>.cat  s      EE}}}yHHgZ%)35AyHHgZ%)35A>>1a&))rK   N)r:  r8   r#   r1  r2  )r]   otherrP  r-   orN  rO  s        @@r    extendzArraysCache.extend  s    
 //""	*& -0

EKK,HI,HDAqc!Qi,HI
 1 153E3EF4<<7 Js   B%c                     t        t        | j                              }| j                  D cg c]
  }|||dz     c}|_        |S c c}w Nr   )r.  r   r#   )r]   r%  r#   r-   s       r    extractzArraysCache.extract  sC    C

O,15<AqsQw'< =s   Ac                 8    t        j                  |      | _        y rA   )r+   r6  r2  )r]   r2  r   s      r    preparezArraysCache.prepare  s    xx(rK   c                      d | _         d | _        y rA   r2  r1  r\   s    r    finalizezArraysCache.finalize  s     rK   c                     | j                   | xj                   |z  c_         | j                  | xj                  |z  c_        y y rA   rZ  )r]   rP   s     r    advancezArraysCache.advance  s<    <<#LLAL(" )rK   rP   c                     | j                   +t        j                  |      }|| j                   d d d f   k\  S | j                  +t        j                  |      }|| j                  d d d f   k  S y rA   )r1  r+   r#  r2  )r]   rP   poss      r    r   zArraysCache.make_mask  sg    ())A,C$++AtG444\\%))A,Cag...rK   c                    t        |d   j                        }t        |      } | |      }t        d |D              r t        j                  dg|z        |_        |S t        |      D ]  t        t        fd|D                    }t        |j                        }||d<   t        j                  ||j                        |<   t        |      D ]  }||      ||      |   ||dz      |S )Nr   c              3   <   K   | ]  }|j                           y wrA   )ro   rD   s     r    rF   z$ArraysCache.merge.<locals>.<genexpr>  s     )&Qqwwy&rG   c              3   4   K   | ]  }|   	|     y wrA   r[   )rE   r-   es     r    rF   z$ArraysCache.merge.<locals>.<genexpr>  s     HVqt7GqtVs   

r   )r   r#   rH   r+   r6  r1  r   nextiterr6   r   r   r   )	rr   r   n_stater   r#   c_initr   r   rc  s	           @r    r   zArraysCache.merge  s    fQioo&KG )&))!#1#'!2ELwA$HVHHIF&EE!Hxxv||4E!H1X!9Q<'&,QilaQU#    rK   c                 $    | j                   d   d u S r|   rI   r\   s    r    ro   zArraysCache.empty  s    zz!}$$rK   c                 :    t        d | j                  D              S )Nc              3   :   K   | ]  }||j                     y wrA   r   rD   s     r    rF   z%ArraysCache.nbytes.<locals>.<genexpr>  s     AZ1=188Zs   )sumr#   r\   s    r    rm   zArraysCache.nbytes  s    ATZZAAArK   rA   )r*   rt   ru   rq   r   r   r   r   rv   r:  r=  r@  r&   rw   rH  rS  rV  rX  r[  r]  r   rx   r   ro   rm   __classcell__)r4  s   @r    r.  r.  R  s    78DI+> 7
 	 	    \\ 78>
)!#3   *% B BrK   r.  c                       e Zd ZdZd Zd Zd Zed        Zej                  d        Zd Z
d Zed	        Zej                  d
        Zd Zed        Zy)ChunkedKVCacher   c                 J    d | _         d | _        d| _        || _        d| _        y r|   )r~   r   rQ   
chunk_sizestart_position)r]   rp  s     r    r   zChunkedKVCache.__init__  s&    	$rK   c                 v   | j                   | j                   j                  d   | j                  k\  r| xj                  | j                   j                  d   | j                  z
  z  c_        | j                   d| j                   d d d f   | _         | j                  d| j                   d d d f   | _        y y y r   )r~   r   rp  rq  r   r\   s    r    maybe_trim_frontzChunkedKVCache.maybe_trim_front  s    99 TYY__Q%74??%J499??1#5#GG		#'7'91"<=DI++cDOO+;+=q&@ADK &K rK   c                    | j                   | j                  z
  }| j                  -||j                  d   z   | j                  j                  d   kD  rf|j                  \  }}}}|j                  d   }| j                  |j                  d   z   dz
  | j                  z  }	|||	| j                  z  |f}
|||	| j                  z  |f}t        j                  |
|j                        }t        j                  ||j                        }| j                  || j                  z  dk7  r6| j                  dd |d d f   | _        | j                  dd |d d f   | _        t        j                  | j                  |gd      | _        t        j                  | j                  |gd      | _        n||c| _        | _        | xj                   |j                  d   z  c_         | j                   | j                  z
  }|| j                  d||d d f<   || j                  d||d d f<   | j                  dd |d d f   | j                  dd |d d f   fS r   )
rQ   rq  r~   r   r   r+   r   r   r   r   )r]   r~   r   r   r   r   r   r   r   r   r   r   r   r   ends                  r    r   zChunkedKVCache.update_and_fetch  s   {{T00099

1!59K K+/::(Az1jaJyy4::a=014BG*g		&9:FG*g		&9:FGHHWdjj1EHHWfll3Eyy$$))#q( $		#uua- 8DI"&++c5D5!m"<DKNNDIIu+=AF	 nndkk5-AJ).&	4;tzz!}$kkD///&*		#tCx"#(.Cc1$%yydsdA&C#qL(AAArK   c                     | j                   | j                  j                  d   k(  r| j                  | j                  fS | j                  dd | j                   d d f   | j                  dd | j                   d d f   fS r   r   r\   s    r    r&   zChunkedKVCache.state  r   rK   c                 ^    |\  | _         | _        | j                   j                  d   | _        y r   r   ra   s     r    r&   zChunkedKVCache.state  r   rK   c                      yr   r[   r\   s    r    rC   zChunkedKVCache.is_trimmable  r   rK   c                 v    t        | j                  | j                  z
  |      }| xj                  |z  c_        |S rA   )r   rQ   rq  r   s     r    rN   zChunkedKVCache.trim  s0    d11115qrK   c                 `    t        t        t        | j                  | j                  f            S rA   )r   r   r   rp  rq  r\   s    r    r'   zChunkedKVCache.meta_state  s#    St0C0CDEFFrK   c                 >    t        t        |      \  | _        | _        y rA   )r   r   rp  rq  ra   s     r    r'   zChunkedKVCache.meta_state   s    /23{,,rK   c                     | j                   d u S rA   r   r\   s    r    ro   zChunkedKVCache.empty$  r   rK   c                 v    | j                   y| j                   j                  | j                  j                  z   S r|   r   r\   s    r    rm   zChunkedKVCache.nbytes'  r   rK   N)r*   rt   ru   r   r   rs  r   rv   r&   rw   rC   rN   r'   ro   rm   r[   rK   r    rn  rn    s    D BB2   \\) )
 G G ; ;! 5 5rK   rn  c                       e Zd Zd Zd Zd Zd Zed        Zej                  d        Zed        Z
e
j                  d        Z
d	 Zd
 Zed        Zd Zd Zd Zd Zd Zed        Zed        Zy)	CacheListc                     || _         y rA   r   )r]   r   s     r    r   zCacheList.__init__/  s	    rK   c                      | j                   |   S rA   r  r?  s     r    r@  zCacheList.__getitem__2  s    {{3rK   c                 :    t        d | j                  D              S )Nc              3   <   K   | ]  }|j                           y wrA   rB   rD   s     r    rF   z)CacheList.is_trimmable.<locals>.<genexpr>6  s     9[1>>#[rG   )rH   r   r\   s    r    rC   zCacheList.is_trimmable5  s    9T[[999rK   c                 J    | j                   D ]  }|j                  |      } S rA   )r   rN   )r]   r   r-   ms       r    rN   zCacheList.trim8  s"    Aq	A rK   c                 T    | j                   D cg c]  }|j                   c}S c c}w rA   )r   r&   r9  s     r    r&   zCacheList.state=  s"    !%-A---s   %c                 N    t        | j                  |      D ]  \  }}||_         y rA   )r8   r   r&   )r]   rb   r-   ss       r    r&   zCacheList.stateA  s"    Q'DAqAG (rK   c                     | j                   D cg c]  }t        |      j                   c}| j                   D cg c]  }|j                   c}fS c c}w c c}w rA   )r   r)   r*   r'   r9  s     r    r'   zCacheList.meta_stateF  sN     (,{{3{!T!W{3#';;/;aQ\\;/
 	
3/s
   AAc                 T    t        | j                  |d         D ]  \  }}||_         y rU  )r8   r   r'   )r]   rb   r-   r  s       r    r'   zCacheList.meta_stateM  s&    QqT*DAqAL +rK   c                 H    | j                   D ]  }|j                  |        yrD  )r   rH  rF  s      r    rH  zCacheList.filterR  s     AHH]# rK   c                 v    t        | j                  |j                        D ]  \  }}|j                  |        y)rJ  N)r8   r   rS  )r]   rQ  r-   rR  s       r    rS  zCacheList.extendY  s,     U\\2DAqHHQK 3rK   c           	           |        }t        fdt        t        d   j                              D              |_        |S )Nc              3      K   | ]@  }d    j                   |   j                  D cg c]  }|j                   |    c}       B yc c}w wr   N)r   r   )rE   r   r-   r   s      r    rF   z"CacheList.merge.<locals>.<genexpr>c  sP      
1 1IQ%%F&CFqqxx{F&CD1 'Ds   $AA	
Ar   )r   r   r   r   )rr   r   r#   s    ` r    r   zCacheList.merge`  s>     
3vay//01
 
 rK   c                 :    t        fd| j                  D         S )Nc              3   @   K   | ]  }|j                          y wrA   )rV  )rE   r-   r%  s     r    rF   z$CacheList.extract.<locals>.<genexpr>j  s     ?;a199S>;s   )r  r   r?  s    `r    rV  zCacheList.extracti  s    ?4;;?@@rK   c                 J    | j                   D ]  } |j                  di |  y )Nr[   )r   rX  )r]   r   r-   s      r    rX  zCacheList.preparel  s!    AAII rK   c                 F    | j                   D ]  }|j                           y rA   )r   r[  r9  s     r    r[  zCacheList.finalizep  s    AJJL rK   c                 :    t        d | j                  D              S )Nc              3   <   K   | ]  }|j                           y wrA   )ri   rD   s     r    rF   z!CacheList.size.<locals>.<genexpr>u  s     1[1668[rG   )maxr   r\   s    r    ri   zCacheList.sizet  s    1T[[111rK   c                 <    | j                   d   j                         S r|   )r   ro   r\   s    r    ro   zCacheList.emptyw  s    {{1~##%%rK   c                 :    t        d | j                  D              S )Nc              3   4   K   | ]  }|j                     y wrA   r   rD   s     r    rF   z#CacheList.nbytes.<locals>.<genexpr>|  s     1[188[   )rk  r   r\   s    r    rm   zCacheList.nbytesz  s    1T[[111rK   c           	          | j                  |       }t        |g| D cg c]#  \  }}}t               |   j                  ||      % c}}}|_        |S c c}}}w rA   )rq   r8   r9   r:   r   )rr   r&   r'   rs   r  r-   r  s          r    r:   zCacheList.from_state~  s[    kk#9<U9PZ9P
9PgaAGIaL##Aq)9P

 

s   (AN)r*   rt   ru   r   r@  rC   rN   rv   r&   rw   r'   rH  rS  rx   r   rV  rX  r[  ri   ro   rm   r:   r[   rK   r    r  r  .  s     :
 . . \\  
 
  $  A 2& 2 2  rK   r  c                     | j                   |   }dd| j                  |z
  z  z   }|d d }t        j                  |      |   ||   z
  |z  }t        j                  | ||      }|S )N).rA   r   r   )r   ndimr+   r#  take_along_axis)r   shiftsr   r   expand_shiftsexpand_indicesr%  rolleds           r    dynamic_rollr    sl    	AW66M"3B'N99Q<'&*??1
DC3T2FMrK   c                       e Zd ZdZdee   fdZd ZdddddZd Z	e
d	        Zej                  d
        Zd Zd ZddedefdZd Zd Zd Zed        Zd Zd Ze
d        Zy)r   r   r1  c                     d| _         d| _        t        j                  |      | _        t        j                  |D cg c]  }|  c}      | _        d| _        d| _        yc c}w )a  
        The BatchKV cache expects inputs to be left-padded.

        E.g. the following prompts:

            [1, 3, 5]
            [7]
            [2, 6, 8, 9]

        Should be padded like so:

            [0, 1, 3, 5]
            [0, 0, 0, 7]
            [2, 6, 8, 9]

        And ``left_padding`` specifies the amount of padding for each.
        In this case, ``left_padding = [1, 3, 0]``.
        Nr   )r~   r   r+   r6  r1  rQ   r  _right_padding)r]   r1  ls      r    r   zBatchKVCache.__init__  sX    & 	HH\2hhL9LqL9:	"  :s   
A!c                    | j                   }| j                  -||j                  d   z   | j                  j                  d   kD  rf|j                  \  }}}}|j                  d   }| j                  |j                  d   z   dz
  | j                  z  }	|||	| j                  z  |f}
|||	| j                  z  |f}t	        j
                  |
|j                        }t	        j
                  ||j                        }| j                  || j                  z  dk7  r6| j                  dd |d d f   | _        | j                  dd |d d f   | _        t	        j                  | j                  |gd      | _        t	        j                  | j                  |gd      | _        n||c| _        | _        | xj                  |j                  d   z  c_	        | xj                   |j                  d   z  c_         || j                  d|| j                   d d f<   || j                  d|| j                   d d f<   | j                  dd | j                   d d f   | j                  dd | j                   d d f   fS r   )
r  r~   r   r   r+   r   r   r   r   rQ   r   s                 r    r   zBatchKVCache.update_and_fetch  s   yy99

1!59K K+/::(Az1jaJyy4::a=014BG*g		&9:FG*g		&9:FGHHWdjj1EHHWfll3Eyy$$))#q( $		#uua- 8DI"&++c5D5!m"<DKNNDIIu+=AF	 nndkk5-AJ).&	4;tzz!}$		TZZ]"	.2		#tdii'*+06C		)1,-yyk		k1,-t{{3$))Q;N/OOOrK   Nr1  r2  right_paddingc                   |V| j                   t        d      t        j                  |      }| xj                  |z  c_        | xj
                  |z  c_        |*t        |      dkD  rt        j                  |      | _        y y y )Nz7Left padding can only be added to an empty BatchKVCacher   )r~   r`   r+   r6  r1  rQ   r  r  r]   r1  r2  r  s       r    rX  zBatchKVCache.prepare  s}    #yy$ M  88L1L-KK<'K$]);a)?"$((="9D *@$rK   c                 (   | j                   | j                   }t        | j                  |d d d f   d      | _        t        | j                  |d d d f   d      | _        | xj                  |z  c_        | xj
                  |z  c_        d | _         y y )Nr   r   )r  r  r~   r   rQ   r1  )r]   paddings     r    r[  zBatchKVCache.finalize  s    *))G$TYY40@qIDI&t{{GAtG4D1MDKKK7"K("&D +rK   c                     | j                   | j                  }}| j                  |j                  d   k  r,|dd | j                  d d f   }|dd | j                  d d f   }||| j                  | j
                  fS r   )r~   r   r  r   rQ   r1  r]   krb   s      r    r&   zBatchKVCache.state  sr    yy$++199qwwqz!#{{A%&A#{{A%&A!T[[$"3"333rK   c                 v    |\  | _         | _        | _        | _        | j                   j                  d   | _        y r   )r~   r   rQ   r1  r   r  ra   s     r    r&   zBatchKVCache.state  s-    AB>	4;T->IIOOA&	rK   c                      yr   r[   r\   s    r    rC   zBatchKVCache.is_trimmable  r   rK   c                     t        | j                  |      }| xj                  |z  c_        | xj                  |z  c_        |S rA   )r   r  rQ   r   s     r    rN   zBatchKVCache.trim  s2    		1		Q	qrK   rP   rR   c                 H    t        |f| j                  | j                  d|S )N)rQ   r1  )r   r  r1  )r]   rP   rR   r   s       r    r   zBatchKVCache.make_mask  s.    !
iid.?.?
CI
 	
rK   c                    | j                   (| j                   |   | _         | j                  |   | _        | j                  |   | _        | j                  |   | _        | j                  j	                         j                         }|dkD  rm| j                   6| j                   d|dddf   | _         | j                  d|dddf   | _        | xj                  |z  c_        | xj                  |z  c_        yy)rE  Nr   .)r~   r   rQ   r1  r   itemr  )r]   rG  min_left_pads      r    rH  zBatchKVCache.filter  s     99 		-0DI++m4DKkk-0 --m< ((,,.335!yy$ IIc<=!&;<	"kk#|}a*?@II%I- rK   c           	        	
 | j                   m|j                   at        j                  | j                  |j                  g      | _        t        j                  | j                  |j                  g      | _        yt        | j                  |j                        	dx}}| j                   4| j                   j                  \  }}| j                  j                  d   |j                   4|j                   j                  \  }}|j                  j                  d   t        ||      
	
fd}t        t        j                  t         ||        ||      f       \  | _         | _        | _        | _        	| _        y)rJ  Nr   r   c                 F   | j                   | j                  }}|g| j                  j                  d   }t	        j
                  g       j                  |	d      }t	        j
                  g       j                  |	d
      }| j                  z
  }|j                  d   z
  |z
  }|dk  r|dd |d d f   }|dd |d d f   }d}|dk7  s|dk7  r4dd||fdg}t	        j                  ||      }t	        j                  ||      }| j                  |z   }||| j                  |fS Nr   r   .)r   r   )
r~   r   rQ   r   r+   r6  reshaper  padr1  )r-   r  rb   Bcleftrightr  r1  DHMmax_idxr   s           r    r  z BatchKVCache.extend.<locals>.pad   s   66188qAyXX^^A&HHRL((Q15HHRL((Q15QVV#Dqwwqz)D0Eqyc6E61n%c6E61n%qyEQJve}f=FF1cNFF1cN>>D0La<//rK   )r~   r+   r   r1  rQ   r  r  r   r   r   r8   r]   rQ  L1L2r   r  r  r  r  r  r   s         @@@@@r    rS  zBatchKVCache.extend  s9    99!3 "0A0A5CUCU/V WD..$++u||)DEDKdii,R99 ))//KAq"a!!!$A::!****KAq"a""1%Ar2;	0 	0& BENNC#d)SZ!89B
>	4;T-> 	rK   c                    t               }| j                  |   j                         }t        j                  | j
                  ||dz   d d || j                  f         |_        t        j                  | j                  ||dz   d d || j                  f         |_        |j
                  j                  d   |_	        |S )Nr   r   )
r   r1  r  r+   
contiguousr~   r  r   r   rQ   )r]   r%  r#   r  s       r    rV  zBatchKVCache.extract8  s    	##C(--/]]499S37]Aw?R-R#ST
}}T[[sQw7TYYCV1V%WXzz''*rK   c                    |D cg c]  }|j                          }}t        |      }|dk(  rt        dgt        |      z        S |D cg c]  }||z
  	 }}t        |      }t        d |D              }t        d |D              }	t        d |D              }
t	        t        d |D                    }t        j                  ||||	f|      }t        j                  ||||
f|      }t        t        ||            D ]  \  }\  }}|j                  |j                  dd |j                  d d f   |||dz   d d |||j                  z   f<   |j                  dd |j                  d d f   |||dz   d d |||j                  z   f<     | |      }||_
        ||_        |xj                  |j                  d	   z  c_        |j                  d	   |_        |S c c}w c c}w )
Nr   c              3   h   K   | ]*  }|j                   |j                   j                  d    , y wrU  r~   r   rD   s     r    rF   z%BatchKVCache.merge.<locals>.<genexpr>K  &     FA1663EQ   22c              3   h   K   | ]*  }|j                   |j                   j                  d    , y wNr   r  rD   s     r    rF   z%BatchKVCache.merge.<locals>.<genexpr>L  &     G&QAFF4Fa&r  c              3   h   K   | ]*  }|j                   |j                   j                  d    , y wr  r   r   rD   s     r    rF   z%BatchKVCache.merge.<locals>.<genexpr>M  '     KFqahh6J"Fr  c              3   b   K   | ]'  }|j                   |j                   j                   ) y wrA   r~   r   rD   s     r    rF   z%BatchKVCache.merge.<locals>.<genexpr>N  "     JVqvv7Iqvv||V   //r   .r   r   )ri   r  r   r   rd  re  r+   r   	enumerater8   r~   rQ   r   r   r  )rr   r   r-   r2  
max_lengthr  r  r   r  DkDvdtr~   r   r   pr#   s                    r    r   zBatchKVCache.merge@  s   %+,V1668V,\
 ?c&k 122+237a:>73KFFFG&GGKFKK$JVJJKxxAz2.b91aR0;"3w#78IAv1vv~3466#zz1:L3MDQUAq1qxx<//056XXc:QXX:q>P5QF1q1u9aQ\!112	 9 G


1%ZZ]
9 - 4s   G
G"c                     | j                   S rA   )r  r\   s    r    ri   zBatchKVCache.size`  s    yyrK   c                     | j                   d u S rA   r   r\   s    r    ro   zBatchKVCache.emptyc  r   rK   c                 v    | j                   y| j                   j                  | j                  j                  z   S r|   r   r\   s    r    rm   zBatchKVCache.nbytesf  r   rK   F)r*   rt   ru   r   r   r   r   r   rX  r[  rv   r&   rw   rC   rN   r,  r   rH  rS  rV  rx   r   ri   ro   rm   r[   rK   r    r   r     s    D#T#Y #6P2 '+D :' 4 4 \\' '
3 
d 

.&+Z  >! 5 5rK   r   c                   >   e Zd ZdZdee   fdZd"dZd Zd Z	d Z
d	 Zdddd
dZd Zed        Zej                   d        Zed        Zej                   d        Zd Zd Zd#dededefdZ	 d$dedee   defdZd Zd Zd Zed        Zd Zd  Zed!        Z y)%r)  r   r1  c                     d | _         d | _        t        j                  |      | _        t        j                  |D cg c]  }|  c}      | _        || _        d| _        d| _        d| _	        d | _
        y c c}w )Nr   F)r~   r   r+   r6  r1  rQ   r   r  _offsetrotated_lengths)r]   r   r1  r  s       r    r   zBatchRotatingKVCache.__init__p  sl    	HH\2hhL9LqL9: 	   :s   
A6Nc                 ^    |dkD  r|d|d d d f   }|t        j                  ||gd      S |S r  )r+   r   )r]   r	  rb   r  s       r    r  zBatchRotatingKVCache._trim  s<    q=#yz1$%A>>1f+A66rK   c                 0   | j                   rt        j                  | j                  | j                   d      | _        t        j                  | j
                  | j                   d      | _        | j                  j                  d   | _        d| _         yy)z:
        Rearrange the cache into temporal order.
        r   r   FN)r  r+   r$  r~   r  r   r   r\   s    r    r  z$BatchRotatingKVCache._temporal_order  sg     <<		DII:A>DI''$++		zBDK		*DI DL	 rK   c                    | j                   || _         || _        n| j                          | j                   j                  d   | j                  kD  rJ| j                   dd | j                  d d f   | _         | j                  dd | j                  d d f   | _        | j
                  t        j                  d| j                  | j
                  z
        }t        | j                   |d d d f   d      | _         t        | j                  |d d d f   d      | _        | xj                  |z  c_
        | xj                  |z  c_        | j                  | j                  z
  dz   }|dkD  r| xj                  |z  c_
        | j                  || j                   |      | _         | j                  || j                  |      | _        | xj                  |j                  d   z  c_        | xj                  |j                  d   z  c_        | j                   j                  d   | _        t        j                  | j                   | j                  | j                  f      | _         | j                   | j                  fS )Nr   .r   r   r   )r~   r   r  r   r  r  r+   maximumrQ   r  r1  r   r  r  depends)r]   r~   r   r$  r	  s        r    r  z#BatchRotatingKVCache._update_concat  s   99DI DK   " yyq!DII- IIc;TYY;&9:	"kk#{{A*=> }}(zz!T[[4==%@A(DDMJ	*4;;QWAN!!T)!t# 		DMM1A5I1}!!Y.!

9dii>DI**YVDDKtzz!}$

1%IIOOA&	 JJtyy4+<+<dkk*JK	yy$++%%rK   c                 *   | j                   t        d      |j                  \  }}}}| j                  }| j                  C|| j                  j                  d   k\  r| j                  j                  d   | j
                  k  r|j                  d   }t        | j                  | j
                  |z
        }	|||	|f}
|||	|f}t        j                  |
|j                        }t        j                  ||j                        }| j                  Qt        j                  | j                  |gd      | _        t        j                  | j                  |gd      | _        n||c| _        | _        || _        | j                  j                  d   | j
                  z
  }|dkD  rh| j                  || j                        | _        | j                  || j                        | _        | j
                  | _        | xj                  |z  c_        | j                  | j
                  k(  rd| _        d| _        | j                   r| xj                  |z  c_        || j                  d| j                  | j                  |z   d d f<   || j                  d| j                  | j                  |z   d d f<   | xj                  |z  c_        | xj"                  |z  c_        | xj                  |z  c_        t        j$                  | j                  | j                  | j"                  f      | _        | j                  | j
                  k  r@| j                  dd | j                  d d f   | j                  dd | j                  d d f   fS | j                  | j                  fS )NzFfinalize() should be called before deocoding with BatchRotatingKVCacher   r   r   r   T.)r  RuntimeErrorr   r  r~   r   r   r   r+   r   r   r   r   r  r  r1  r  rQ   r  r  s                  r    r  z%BatchRotatingKVCache._update_in_place  s   ==$X  (,zz$:q*||99DIIOOA&&499??1+=+MaJ499dmmd&:;H*h
;G*h
;GHHWdjj1EHHWfll3Eyy$NNDIIu+=AF	 nndkk5-AJ).&	4;DI IIOOA&6	q=

9dii8DI**Y<DKDI* 99%DLDI<<" 8<		#tyy499q=0!349?CTYY]2A56q		Q	 JJtyy4+<+<dkk*JK	 <<$--'		#~~q01C4<<23  yy$++%%rK   c                 n    |j                   d   dk(  r| j                  ||      S | j                  ||      S r  r  r   s      r    r   z%BatchRotatingKVCache.update_and_fetch  r  rK   r  c                &   |V| j                   t        d      t        j                  |      }| xj                  |z  c_        | xj
                  |z  c_        |7t        |      dkD  r(t        j                  |      | j
                  z   | _        y y y )Nz?Left padding can only be added to an empty BatchRotatingKVCacher   )r~   r`   r+   r6  r1  rQ   r  r  r  s       r    rX  zBatchRotatingKVCache.prepare  s    #yy$ U  88L1L-KK<'K$]);a)?HHW-;DM *@$rK   c                 j   | j                   t        j                  d| j                  | j                   z
        }t	        | j
                  |d d d f   d      | _        t	        | j                  |d d d f   d      | _        | xj                  |z  c_        | xj                  |z  c_        d | _         y y )Nr   r   r   )r  r+   r  rQ   r  r~   r   r1  )r]   r$  s     r    r[  zBatchRotatingKVCache.finalize  s    ==$::at}}!<=D$TYYQWAFDI&t{{DDMJDK%KK4K DM %rK   c                     | j                   | j                  }}| j                  |j                  d   k  r,|dd | j                  d d f   |dd | j                  d d f   }}||| j                  | j
                  fS r   )r~   r   r  r   rQ   r1  r  s      r    r&   zBatchRotatingKVCache.state  sq    yy$++1<<!''!*$S.DLL.!+,a^t||^Q0F.GqA!T[[$"3"333rK   c                 :    |\  | _         | _        | _        | _        y rA   r~   r   rQ   r1  ra   s     r    r&   zBatchRotatingKVCache.state  s    AB>	4;T->rK   c           	          t        t        t        | j                  | j                  | j
                  | j                  f            S rA   )r   r   r   r   r  r  r  r\   s    r    r'   zBatchRotatingKVCache.meta_state  s-    St}}dllDIIt||TUVVrK   c                 v    t        t        |d d       \  | _        | _        | _        t        |d         | _        y r  )r   r   r   r  r  r,  r  ra   s     r    r'   zBatchRotatingKVCache.meta_state  s6    14bqE2
.t|TY AaDzrK   c                 4    | j                   | j                  k  S rA   )r  r   r\   s    r    rC   z!BatchRotatingKVCache.is_trimmable%  s    ||dmm++rK   c                     t        | j                  |      }| xj                  |z  c_        | xj                  |z  c_        | xj                  |z  c_        |S rA   )r   r  r  rQ   r   s     r    rN   zBatchRotatingKVCache.trim(  s@    a 		Q	qrK   r   r   r   c                     t        d      )Nz%BatchRotatingKVCache Quantization NYIrk   r   s      r    r   z!BatchRotatingKVCache.to_quantized/  s    !"IJJrK   rP   rS   rR   c                    | j                   }|xs | j                  }t        | j                  dz
  | j                        }t	        j
                  ||z         }|rt	        j
                  |||z         n|}|d d d f   }|d    }||k\  }||||z   k  z  }| j                  | j                  z
  t        |dkD        z   x}	dkD  r||	z
  }|dk(  xr' | j                  xs | j                  | j                  k\  }
|
r|dz
  }||t	        j                  |d      k\  z  }|
r8| j                  }|| j                  k\  rd}t	        j                  ||dz   d      }|S )Nr   r   )r   r   r   r   )r"  r   )r1  r   r   r  r+   r#  r  r   r  expand_dimsr$  )r]   rP   rS   rR   r1  rQ   rindslindsr'  r	  r  r%  s               r    r   zBatchRotatingKVCache.make_mask2  sG    ((!2T]]T]]Q&5		&1*%17		&&1*-Uagd~+++T]]2SQZ??I1D')3Lq&IdllHdii4==.H'!+Lu|Y GGH))Cdmm#774sQwR8DrK   c                     | j                   (| j                   |   | _         | j                  |   | _        | j                  |   | _        | j                  |   | _        yrD  r  )r]   rG  s     r    rH  zBatchRotatingKVCache.filterO  sQ     99 		-0DI++m4DKkk-0 --m<rK   c           	        	
 | j                   m|j                   at        j                  | j                  |j                  g      | _        t        j                  | j                  |j                  g      | _        y| j
                  |j
                  k7  s| j                  |j                  k7  r | j                          |j                          t        | j                  |j                        	dx}}| j                   4| j                   j                  \  }}| j                  j                  d   |j                   4|j                   j                  \  }}|j                  j                  d   t        ||      
	
fd}t        t        j                  t         ||        ||      f       \  | _         | _
        | _        | _        	| _        t        | j                  |j                        | _        y)rJ  Nr   r   c                 F   | j                   z
  }| j                  | j                  }}|g| j                  j                  d   }t        j                  g       j                  |	d      }t        j                  g       j                  |	d
      }|j                  d   z
  |z
  }|dk  r|dd |d d f   }|dd |d d f   }d}|dk7  s|dk7  r4dd||fdg}t        j                  ||      }t        j                  ||      }| j                  |z   }||| j                  |fS r  )
r  r~   r   rQ   r   r+   r6  r  r  r1  )r-   r  r  rb   r  r  r  r1  r  r  r  r  r   s           r    r  z(BatchRotatingKVCache.extend.<locals>.padp  s   QVV#D66188qAyXX^^A&HHRL((Q15HHRL((Q15qwwqz)D0Eqyc6E61n%c6E61n%qyEQJve}f=FF1cNFF1cN>>D0La<//rK   )r~   r+   r   r1  rQ   r  r  r  r  r   r   r   r8   r  r  s         @@@@@r    rS  zBatchRotatingKVCache.extendY  s    99!3 "0A0A5CUCU/V WD..$++u||)DEDKLLEMM)dii5::.E  "!!#dii,R99 ))//KAq"a!!!$A::!****KAq"a""1%Ar2;	0 	0& BENNC#d)SZ!89B
>	4;T-> 	4<<7rK   c                    t        j                  | j                  | j                         t	        | j
                        }t        d| j                  j                         |         }| j                  j                         |   }| j                  ||dz    |_        | j                  ||dz    |_	        | j                  |_
        | j                  rut        j                  |j                  | j                   d      |_        t        j                  |j                  | j                   d      |_	        | j
                  |_
        t        j                  |j                  d d d d ||j                  f         |_        t        j                  |j                  d d d d ||j                  f         |_	        ||_        |j                  j                  d   |_
        |S )Nr   r   r   r   )r+   evalr1  rQ   r   r   r  tolistr~   r   r  r  r$  r  r   )r]   r%  r#   r  rQ   s        r    rV  zBatchRotatingKVCache.extract  sX   
!!4;;/.a**113C89##%c*YYsS1W-
{{3q1YY
<<diiZa@EJ775<<$))!DELEJ]]5::aGejj4H.H#IJ
}}U\\!Q%**8L2L%MNZZ%%a(
rK   c                    t        fdD              st        d      D cg c]  }|j                   }}D cg c]  }|j                          }}t	        |      }|dk(  r# | d   j
                  dgt              z        S |D cg c]  }||z
  	 }}t              }t	        d D              }	t	        d D              }
t	        d D              }t        t        d D                    }t        j                  ||	||
f|      }t        j                  ||	||f|      }t        t        ||            D ]  \  }\  }}}|j                  |j                  |j                        d	| d d d f   |||d
z   d d |||z   f<   |j                  |j                        d	| d d d f   |||d
z   d d |||z   f<     | d   j
                  |      }||_        ||_        t        j                   |      |_        |j"                  d   |_        |j"                  d   |_        |S c c}w c c}w c c}w )Nc              3   V   K   | ]   }|j                   d    j                   k(   " ywr  )r   )rE   r-   r   s     r    rF   z-BatchRotatingKVCache.merge.<locals>.<genexpr>  s%     DV1::!3!33Vs   &)zEBatchRotatingKVCache can only merge caches with the same maximum sizer   c              3   h   K   | ]*  }|j                   |j                   j                  d    , y wrU  r  rD   s     r    rF   z-BatchRotatingKVCache.merge.<locals>.<genexpr>  r  r  c              3   h   K   | ]*  }|j                   |j                   j                  d    , y wr  r  rD   s     r    rF   z-BatchRotatingKVCache.merge.<locals>.<genexpr>  r  r  c              3   h   K   | ]*  }|j                   |j                   j                  d    , y wr  r  rD   s     r    rF   z-BatchRotatingKVCache.merge.<locals>.<genexpr>  r  r  c              3   b   K   | ]'  }|j                   |j                   j                   ) y wrA   r  rD   s     r    rF   z-BatchRotatingKVCache.merge.<locals>.<genexpr>  r  r  r   .r   r   )rH   r`   rQ   ri   r  r   r   rd  re  r+   r   r  r8   r~   r  r   r6  r   r  r  )rr   r   r-   offsetsr2  r  r  r  r   r  r  r  r  r~   r   r   r  r#   s    `                r    r   zBatchRotatingKVCache.merge  s7   DVDDW  &,,V188V,%+,V1668V,\
 ?vay))A3V+<==+237a:>73KFFFG&GGKFKK$JVJJKxxAz2.b91aR0;%c'7F&CDLAy1avv~,-,=,=aff,EcA23PQk,RDQUAq1q5y()./.?.?.I#PQrsTU+.VF1q1u9aQU*+	 E F1I&&0
xx(ZZ]


1= -, 4s   H5H:H?c                 B    t        | j                  | j                        S rA   )r   r  r   r\   s    r    ri   zBatchRotatingKVCache.size  s    4<<//rK   c                     | j                   d u S rA   r   r\   s    r    ro   zBatchRotatingKVCache.empty  r   rK   c                 v    | j                   y| j                   j                  | j                  j                  z   S r|   r   r\   s    r    rm   zBatchRotatingKVCache.nbytes  r   rK   rA   r  rg   )!r*   rt   ru   r   r   r   r   r  r  r  r  r   rX  r[  rv   r&   rw   r'   rC   rN   r   r   r   r,  r   rH  rS  rV  rx   r   ri   ro   rm   r[   rK   r    r)  r)  m  s7   DtCy  !%&N9&v1
 '+D <! 4 4 \\C C W W " ",Ks Ks KCS K OT#+C=GK:=.8`$ $ $L0! 5 5rK   r)  c                   D    e Zd ZdZdZg fdZd Zed        Zed        Z	y)TokenBufferzA simple token buffer that can be efficiently appended to in a similar
    fashion to the KVCache.

    Perhaps these could share some logic in the future.
    r   c                 x    t        j                  |t         j                        | _        t	        |      | _        y r   )r+   r6  int32_bufferr   _size)r]   tokenss     r    r   zTokenBuffer.__init__  s#    xxbhh7[
rK   c                    | j                   }|t        |      z   }|| j                  z   dz
  | j                  z  | j                  z  }|| j                  j                  kD  r`t        j                  | j                  t        j                  || j                  j                  z
  t
        j                        g      | _        || j                  || || _         | j                  d | S rL  )	r  r   r   r  ri   r+   r   r   r  )r]   r  startru  r  s        r    r   zTokenBuffer.update_and_fetch  s    

c&k!499_q(TYY6$))Cdll'''>>rxx4<<3D3D(DBHHUVDL #)U3
||DS!!rK   c                     | j                   S rA   )r  r\   s    r    r&   zTokenBuffer.state  s    ||rK   c                 4    | j                   d | j                   S rA   )r  r  r\   s    r    r  zTokenBuffer.tokens  s    ||Ldjj))rK   N)
r*   rt   ru   r   r   r   r   rv   r&   r  r[   rK   r    r  r    sC     D  !"   * *rK   r  c                   d    e Zd ZU eed<   eee      ed<   eee      ed<   eee      ed<   eed<   y)PromptTrieResultr   exactshorterlongercommon_prefixN)r*   rt   ru   r   __annotations__r   r   r   r[   rK   r    r  r    s:    JDId3i  T#YrK   r  c                       e Zd Zd Zdedee   defdZdedee   fdZdedee   fdZ	dedee   fdZ
dedee   d	efd
Zy)
PromptTriec                     i | _         y rA   _trier\   s    r    r   zPromptTrie.__init__  s	    
rK   r   r  r<  c                     || j                   vri | j                   |<   | j                   |   }|D ]  }||vri ||<   ||   } |j                  dd       }||d<   |S N	__value__)r%  get)r]   r   r  r<  currenttokr   s          r    addzPromptTrie.add   so    

" "DJJu**U#C'!!clG  {{;-$rK   c                 B    | j                   |   }|D ]  }||   }	 |d   S r'  r$  )r]   r   r  r*  r+  s        r    r)  zPromptTrie.get  s.    **U#CclG {##rK   c                    | j                   |   g}|D ]  }|j                  |d   |           |d   j                  d      }t        t	        |      dd      D ]+  }||   }||dz
     }||dz
     }t	        |      dkD  r |S ||= - |S )Nr   r(  r   r   )r%  r  popr   r   )	r]   r   r  pathr+  r<  r   nodeparents	            r    r/  zPromptTrie.pop  s    

5!"CKKR& R[)s6{Ar*A7D!a%[FQ-C4y1} s + rK   c                     g }| j                   |   }t        |      D ]0  \  }}d|v r"|j                  ||j                  d      f       ||   }2 |S r'  )r%  r  r  r/  )r]   r   r  r   r*  r   r+  s          r    pop_prefixeszPromptTrie.pop_prefixes!  sZ    **U#'FAsg%q'++k":;<clG ( rK   r   c                    || j                   vrt        |d d d d      S | j                   |   }|sd|v rt        |g d d d      S d}d}|t        |      k  r0||   |v r)|||      }d|v r|}|dz  }|t        |      k  r||   |v r)|t        |      dz
  cxk(  rdk\  rn nt        ||d d d      S d }|dkD  r|d |dz    }d }|}|dkD  rd }	|g fg}
|
rp|
j                         \  }}d|v r|	t        |      t        |	      k  r=|}	n:|	t        |      t        |	      k  r!|D ]  }|
j	                  ||   ||gz   f        |
rp|d | |	z   }t        |d |||      S )Nr   r(  r   r   )r%  r  r   r/  r  )r]   r   r  r*  
last_indexindexr  r  r  beststackextrar+  s                r    searchzPromptTrie.search*  s   

"#E4tQ??**U#+0#E2tT1== 
c&k!fUmw&>fUm,Gg%"
QJE	 c&k!fUmw&> Vq-A-#E64qAA >-zA~.G 19Dr]OE!&')|s5zCI'=$\SZ#d)%;&gclESEM%BC  '  FU^d*FtWfmLLrK   N)r*   rt   ru   r   r   r   r   r,  r)  r/  r4  r  r;  r[   rK   r    r"  r"    s     d3i  $ $d3i $ d3i # tCy *MC *Mc *M7G *MrK   r"  c            	           e Zd Ze G d d             Z G d d      ZddedefdZd Ze	d	        Z
d
edee   fdZddd
edee   dee   defdZddddee   dee   fdZd Zy)LRUPromptCachec                   2    e Zd ZU ee   ed<   eed<   eed<   y)LRUPromptCache.CacheEntryprompt_cacherm   
cache_typeN)r*   rt   ru   r   r   r   r   r   r[   rK   r    
CacheEntryr?  X  s    3irK   rB  c                   d    e Zd Zg dfdee   fdZd Zddedee   defdZdedee   fd	Z	d
 Z
y)LRUPromptCache.CacheOrder)	assistantusersystemorderingc                 X    || _         |D ci c]  }|t                c}| _        y c c}w rA   )	_orderingr   _lrus)r]   rH  r  s      r    r   z"LRUPromptCache.CacheOrder.__init___  s(    %DN.67h!UW*h7DJ7s   'c                 V    t        d | j                  j                         D              S )Nc              3   2   K   | ]  }t        |        y wrA   )r   )rE   lrus     r    rF   z4LRUPromptCache.CacheOrder.__len__.<locals>.<genexpr>d  s     ?+>Cs3x+>s   )rk  rK  r   r\   s    r    __len__z!LRUPromptCache.CacheOrder.__len__c  s     ?4::+<+<+>???rK   r   r  rA  c                 D    | j                   |   j                  ||f       y rA   )rK  r  r]   r   r  rA  s       r    pushzLRUPromptCache.CacheOrder.pushf  s    JJz"))5&/:rK   c                     | j                   D ]$  }	 | j                  |   j                  ||f        y  y # t        $ r Y 3w xY wrA   )rJ  rK  remover`   rQ  s       r    rT  z LRUPromptCache.CacheOrder.removei  sG    "nn
JJz*115&/B - " s    5	A Ac                 f   d}|dz   t        | j                        k  r| j                  | j                  |      }| j                  | j                  |dz         }|r't        |      t        |      k\  r|j                         S |dz  }|dz   t        | j                        k  rj                         S r8  )r   rJ  rK  popleft)r]   r   lru_alru_bs       r    r/  zLRUPromptCache.CacheOrder.popq  s    Aa%#dnn--

4>>!#45

4>>!a%#89SZ3u:5 ==?*Q a%#dnn-- ==?"rK   N)rE  )r*   rt   ru   r   r   r   rO  r   rR  rT  r/  r[   rK   r    
CacheOrderrD  ^  sU    1P 	8T#Y 	8	@	;c 	;49 	;# 	;	 	T#Y 		#rK   rY  r   	max_bytesc                     || _         || _        t               | _        t        j                         | _        d| _        | j                  j                  D ci c]  }|d c}| _	        y c c}w r|   )
r   rZ  r"  r%  r=  rY  _lru_n_bytesrJ  _n_bytes_by_type)r]   r   rZ  r  s       r    r   zLRUPromptCache.__init__{  sZ     "\
"--/	/3yy/B/B C/B!A/B C Cs   
A)c                 ,    t        | j                        S rA   )r   r\  r\   s    r    rO  zLRUPromptCache.__len__  s    499~rK   c                     | j                   S rA   )r]  r\   s    r    rm   zLRUPromptCache.nbytes  s    }}rK   r   r  c                    | j                   j                  ||      }|j                  Q| j                   j                  |j                  |j                        }t        j                  |j                        g fS |j                  t        |j                        nd}|j                  |j                  |kD  r| j                   j                  |j                  |j                        }t        |j                        rlt        j                  |j                        }t        t        |      dz
  |j                        }t        |j                        |z
  }t        ||       |||d  fS |dkD  rT| j                   j                  |j                  |j                        }t        j                  |j                        ||d  fS d |fS r8  )r%  r;  r  r)  r   copydeepcopyr@  r  r   r  r  rJ   r   rO   )	r]   r   r  resultcache_entryshort_lengthr#   prefixnum_to_trims	            r    fetch_nearest_cachez"LRUPromptCache.fetch_nearest_cache  s]   ""5&1<<#**..v||DK==!9!9:B>>.4nn.Hs6>>*a==$)=)=)L**..v}}EK$[%=%=>k&>&>?S[1_f.B.BC!&--069!%5fVWo--!**..v~~FK==!9!9:F<=<QQQV|rK   rE  )rA  r@  rA  c                   t         j                  |t        d |D              |      }| xj                  |j                  z  c_        | j
                  |xx   |j                  z  cc<   | j                  j                  |||      }|f| xj                  |j                  z  c_        | j
                  |j                  xx   |j                  z  cc<   | j                  j                  ||       | j                  j                  |||       t        |      r| j                  j                  ||      D ]n  \  }}| xj                  |j                  z  c_        | j
                  |j                  xx   |j                  z  cc<   | j                  j                  ||d |        p t        | j                        | j                  kD  r| j                  j!                         \  }}| j                  j!                  ||      }| xj                  |j                  z  c_        | j
                  |j                  xx   |j                  z  cc<   | j                  | j"                  kD  r| j                  j!                         \  }}| j                  j!                  ||      }| xj                  |j                  z  c_        | j
                  |j                  xx   |j                  z  cc<   | j                  | j"                  kD  ry y )Nc              3   4   K   | ]  }|j                     y wrA   r   rD   s     r    rF   z.LRUPromptCache.insert_cache.<locals>.<genexpr>  s     =1ahhr  )r=  rB  rk  r]  rm   r^  r%  r,  rA  r\  rT  rR  rJ   r4  r   r   r/  rZ  )r]   r   r  r@  rA  entryr   
prefix_lens           r    insert_cachezLRUPromptCache.insert_cache  s*    ))#===z

 	%j)U\\9)zz~~eVU3MMT[[(M!!$//2dkkA2IIUF+		ufj1 !.%)ZZ%<%<UF%K!
E-%%e&6&675<<G7		  {
(;< &L tyy>DMM) IIMMOME6JJNN5&1EMMU\\)M!!%"2"23u||C3mmdnn, IIMMOME6JJNN5&1EMMU\\)M!!%"2"23u||C3	 mmdnn,rK   Nn_sequencesn_bytesrp  rq  c                   |t        d|      nd}|t        d|      nd}t        | j                        |kD  r| j                  j                         \  }}| j                  j                  ||      }| xj
                  |j                  z  c_        | j                  |j                  xx   |j                  z  cc<   t        | j                        |kD  r| j
                  |kD  r| j                  j                         \  }}| j                  j                  ||      }| xj
                  |j                  z  c_        | j                  |j                  xx   |j                  z  cc<   | j
                  |kD  ry y )Nr               )	r  r   r\  r/  r%  r]  rm   r^  rA  )r]   rp  rq  r   r  rl  s         r    trim_tozLRUPromptCache.trim_to  s    .9-Dc![)'%,%8#a/g$))n{* IIMMOME6JJNN5&1EMMU\\)M!!%"2"23u||C3	 $))n{*
 mmg% IIMMOME6JJNN5&1EMMU\\)M!!%"2"23u||C3	 mmg%rK   c                     i }| j                   j                  D ]7  }t        | j                   j                  |         | j                  |   d||<   9 |S )Nro  )r\  rJ  r   rK  r^  )r]   rd  rA  s      r    stats_by_typezLRUPromptCache.stats_by_type  sQ    ))--J"499??:#>?00<"F: .
 rK   )
   rs  )r*   rt   ru   r   rB  rY  r   r   rO  rv   rm   r   r   ri  r   rn  r   rt  rv  r[   rK   r    r=  r=  W  s      
# #:D Dc D   d3i 8 &)D)D S	)D 3i	)D )DX /3TD&smD=Ec]D"rK   r=  rA   r  )/rb  collectionsr   dataclassesr   typingr   r   r   r   mlx.corecorer+   mlx.nnnn	mlx.utilsr	   r
   r   r   baser   Moduler   r!   r   r2   r>   r,  rJ   rO   rW   rY   rz   r   r   r   r.  rn  r  r  r   r)  r  r  r"  r=  r[   rK   r    <module>r     s     ! , ,   I I $
 "&6996#6 
#Y68 TV ? ?T#Y ?$sCx. ?&40c 0t 02T#Y 2C 2DI 2&



'+
:B3-
0 0f35 35lZSz ZSzR5j R5ju5j u5pFB* FBRP5Z P5fV
 VrZ5: Z5z_5: _5D!* !*H   XM XMvL LrK   