
    ^NjfS              
       R   U d dl Z d dlmZmZmZmZmZ d dlZd dlZ	d dl
mZ d dlmZ d dlmZ d dlmZmZ d dlmZ d dlmZmZ d d	lmZmZ d d
lmZ d dlmZmZm Z   eddddd ed      dgd      gZ!e"e   e#d<    G d deee         Z$ G d dee         Z% G d de e         Z&y)    N)AnyIterableTypeOptionalSequence)Encoding)Image)
ImageInput)DenseModelDescriptionModelSource)OnnxOutputContext)
NumpyArrayOnnxProvider)define_cache_dir
iter_batch)&LateInteractionMultimodalEmbeddingBase)OnnxMultimodalModelTextEmbeddingWorkerImageEmbeddingWorkerzQdrant/colmodernvbert   zIThe late-interaction version of ModernVBERT, CPU friendly, English, 2025.mitg      ?)hfprocessor_config.jsonz
model.onnx)modeldimdescriptionlicense
size_in_GBsourcesadditional_files
model_filesupported_colmodernvbert_modelsc                       e Zd ZdZdZdZ	 	 	 	 	 	 	 	 d.dedee   dee   dee	e
      ded	eee      d
edee   dee   def fdZedee   fd       Zd/dZdeeef   dedeeef   fdZdedee   fdZdee   dedee   fdZ	 	 d0deee   z  dedededef
dZdee   dedefdZedeee      deeeeeef   f   fd       Z deee!jD                  f   dedeeef   fd Z#ed!edeeef   fd"       Z$defd#Z%d$ed%edefd&Z&d$ed%ede!jD                  fd'Z'dedee   fd(Z(	 	 d1deee   z  ded)ee   dedee   f
d*Z)	 	 d2deee   z  ded)ee   dedee   f
d+Z*ede+e,e      fd,       Z-ede+e.e      fd-       Z/ xZ0S )3ColModernVBERTz
    The ModernVBERT/colmodernvbert model implementation. This model uses
    bidirectional attention, which proves to work better for retrieval.

    See: https://huggingface.co/ModernVBERT/colmodernvbert
    zM<|begin_of_text|>User:<image>Describe the image.<end_of_utterance>
Assistant:z<end_of_utterance>
model_name	cache_dirthreads	providerscuda
device_ids	lazy_load	device_idspecific_model_pathkwargsc
                 `   t        |   |||fi |
 || _        || _        | j	                  |
      | _        || _        || _        d| _        ||| _        n | j                  | j                  d   | _        | j                  |      | _
        t        t        |            | _        |	| _        | j                  | j                  | j                  | j                   | j                        | _        d| _        d| _        d| _        d| _        d| _        | j                  s| j/                          yy)a  
        Args:
            model_name (str): The name of the model to use.
            cache_dir (str, optional): The path to the cache directory.
                                       Can be set using the `FASTEMBED_CACHE_PATH` env variable.
                                       Defaults to `fastembed_cache` in the system's temp directory.
            threads (int, optional): The number of threads single onnxruntime session can use. Defaults to None.
            providers (Optional[Sequence[OnnxProvider]], optional): The list of onnxruntime providers to use.
                Mutually exclusive with the `cuda` and `device_ids` arguments. Defaults to None.
            cuda (bool, optional): Whether to use cuda for inference. Mutually exclusive with `providers`
                Defaults to False.
            device_ids (Optional[list[int]], optional): The list of device ids to use for data parallel processing in
                workers. Should be used with `cuda=True`, mutually exclusive with `providers`. Defaults to None.
            lazy_load (bool, optional): Whether to load the model during class initialization or on demand.
                Should be set to True when using multiple-gpu and parallel encoding. Defaults to False.
            device_id (Optional[int], optional): The device id to use for loading the model in the worker process.

        Raises:
            ValueError: If the model_name is not in the format <org>/<model> e.g. BAAI/bge-base-en.
        Nr   )local_files_onlyr-   )super__init__r(   r+   _select_exposed_session_options_extra_session_optionsr*   r)   r,   _get_model_descriptionmodel_descriptionstrr   r&   _specific_model_pathdownload_model_local_files_only
_model_dirmask_token_idpad_token_idimage_seq_lenmax_image_size
image_sizeload_onnx_model)selfr%   r&   r'   r(   r)   r*   r+   r,   r-   r.   	__class__s              /Users/ahmed/devFolder/Ultron/claude-voice/.venv/lib/python3.12/site-packages/fastembed/late_interaction_multimodal/colmodernvbert.pyr2   zColModernVBERT.__init__2   s$   B 	YB6B""&*&J&J6&R# %	 )- &DN__(!__Q/DN!%!<!<Z!H-i89$7!--""NN!33 $ 9 9	 . 
 " ,0-1)-~~  "     returnc                     t         S )zLists the supported models.

        Returns:
            list[DenseModelDescription]: A list of DenseModelDescription objects containing the model information.
        )r"   clss    rD   _list_supported_modelsz%ColModernVBERT._list_supported_modelsv   s
     /.rE   c           	      :   | j                  | j                  | j                  j                  | j                  | j
                  | j                  | j                  | j                         | j                  dz  }t        |      5 }t        j                  |      }|j                  dd      | _        d d d        | j                  dz  }t        |      5 }t        j                  |      }|j                  di       j                  dd      | _        d d d        | j                  d	z  }t        |      5 }t        j                  |      }|j                  d
i       }|j                  dd      | _        d d d        y # 1 sw Y   xY w# 1 sw Y   wxY w# 1 sw Y   y xY w)N)	model_dirr!   r'   r(   r)   r,   extra_session_optionsr   r>   @   zpreprocessor_config.jsonr?   longest_edgei   zconfig.jsonvision_configr@   )_load_onnx_modelr;   r6   r!   r'   r(   r)   r,   r4   openjsonloadgetr>   r?   r@   )	rB   processor_config_pathfprocessor_configpreprocessor_config_pathpreprocessor_configconfig_pathmodel_configrP   s	            rD   rA   zColModernVBERT.load_onnx_model   s[   oo--88LLnnnn"&"="= 	 	
 !%2I I'(A#yy|!1!5!5or!JD ) $(??5O#O *+q"&))A,"5"9"9:JB"O"S"S#D , oo5+!99Q<L(,,_bAM+//cBDO  )(
 ,+ s$   -E9=F1?F9FFF
onnx_inputc                     |d   j                   \  }}t        j                  ||d| j                  | j                  ft        j                        }||d<   |S )  
        Post-process the ONNX model output to convert it into a usable format.

        Args:
            output (OnnxOutputContext): The raw output from the ONNX model.

        Returns:
            Iterable[NumpyArray]: Post-processed output as NumPy arrays.
        	input_ids   dtypepixel_values)shapenpzerosr@   float32)rB   r]   r.   
batch_size
seq_lengthempty_image_placeholders         rD   _preprocess_onnx_text_inputz*ColModernVBERT._preprocess_onnx_text_input   sW     ",K!8!>!>
J.0hhQI**/
 &=
>"rE   outputc                     |j                   S )r_   )model_outputrB   rm   s     rD   _post_process_onnx_text_outputz-ColModernVBERT._post_process_onnx_text_output   s     """rE   	documentsc                     |D cg c]  }|| j                   dz  z    }}| j                  j                  |      }|S c c}w )N
   )QUERY_AUGMENTATION_TOKEN	tokenizerencode_batch)rB   rr   r.   docaugmented_queriesencodeds         rD   tokenizezColModernVBERT.tokenize   sI    QZ[QZ#S4#@#@2#EEQZ[..--.?@ \s   =textsri   include_extensionc           
      ~   t        | d      r| j                  | j                          d}t        |t              r|gn|}| j
                  J |r| j                  n| j
                  j                  }t        ||      D ]7  }|t         ||      D cg c]  }t        |j                         c}      z  }9 |S c c}w )Nr   r   )hasattrr   rA   
isinstancer7   rv   r{   rw   r   sumattention_mask)	rB   r|   ri   r}   r.   	token_numtokenize_funcbatchencodings	            rD   token_countzColModernVBERT.token_count   s     tW%);  "	%eS1u~~)))):@[@[z2E=Y^K_`K_xc("9"9:K_`aaI 3 as   B:imagesc           	         t        j                         5 }|D cg c]B  }t        |t        j                        s$|j	                  t        j
                  |            n|D }}| j                  J d       | j                  |      }| j                  |      \  }}}	d d d        d}
 | j                  |
fi |}
| j                  j                  d |
      }t        |d   |	      S c c}w # 1 sw Y   SxY w)NzProcessor is not initialized)rd   r   r   )ro   r   metadata)
contextlib	ExitStackr   r	   enter_contextrR   	processor_process_nested_patches_preprocess_onnx_image_inputr   runr   )rB   r   r.   stackimageimage_files	processedrz   r   r   r]   ro   s               rD   onnx_embed_imagezColModernVBERT.onnx_embed_image   s   !!#u
 $	 $E "%5 ##EJJu$56 $	   >>-M/MM-{3I040L0LY0W-G^X $ '.P
6T66zLVL
zz~~dJ7 %a)
 	
 $#s   C.AC)!;C.)C..C7r   c                    | D cg c]  }t        |       }}t        |      }| d   d   j                  \  }}}t        |       }t        j                  |||||f| d   d   j
                        }t        j                  ||ft        j                        }	t        |       D ]&  \  }
}t        |      D ]  \  }}|||
|f<   d|	|
|f<    ( d|i}||	|fS c c}w )a  
        Process nested image patches (from ImageSplitter).

        Args:
            processed: List of patch lists, one per image [[img1_patches], [img2_patches], ...]

        Returns:
            tuple: (encoded array, attention_mask, metadata)
                - encoded: (batch_size, max_patches, C, H, W)
                - attention_mask: (batch_size, max_patches) with 1 for real patches, 0 for padding
                - metadata: Dict with 'patch_counts' key
        r   rb      patch_counts)lenmaxre   rf   rg   rc   int64	enumerate)r   patchesr   max_patcheschannelsheightwidthri   rz   r   ijpatchr   s                 rD   r   z&ColModernVBERT._process_nested_patches   s      5>>IGI>,' #,A,q/"7"7&%^
 ((h>iPQlSToF[F[

 :{";288L $I.JAw%g.5 %1'(q!t$ / /
 #L100- ?s   Cc                    |d   }|d   }|j                   d   }g }t        |      D ]Z  }t        t        j                  ||               }| j                  |      \  }	}
| j                  |	|
      }|j                  |       \ t        d |D              }| j                  j                  d   }| j                  j                  d   }t        j                  ||f|t        j                        }t        j                  ||ft        j                        }t        |      D ]?  \  }}t        |      }|dk(  r||z
  }||||d	f<   d
|||d	f<   .|||d	|f<   d
||d	|f<   A ||d<   ||d<   |S )aQ  
        Add text input placeholders for image data, following Idefics3 processing logic.

        Constructs input_ids dynamically based on the actual number of image patches,
        using the same token expansion logic as Idefics3Processor.

        Args:
            onnx_input: Dict with 'pixel_values' (batch, num_patches, C, H, W)
                        and 'attention_mask' (batch, num_patches) indicating real patches
            **kwargs: Additional arguments

        Returns:
            Updated onnx_input with 'input_ids' and updated 'attention_mask' for token sequence
        r   rd   r   c              3   2   K   | ]  }t        |        y wN)r   ).0idss     rD   	<genexpr>z>ColModernVBERT._preprocess_onnx_image_input.<locals>.<genexpr><  s     :/3c#h/s   	directionpad_idrb   leftNr   r`   )re   rangeintrf   r   _compute_rows_cols_from_patches_build_input_ids_for_imageappendr   rv   paddingfullr   rg   r   r   )rB   r]   r.   patch_attention_maskrd   ri   batch_input_idsr   patch_countrowscolsr`   max_lenpadding_directionr=   padded_input_idsr   seq_len	start_idxs                      rD   r   z+ColModernVBERT._preprocess_onnx_image_input  s   &  **:;!.1!''*
 z"Abff%9!%<=>K ==kJJD$ 77dCI""9- # :/:: !NN22;?~~--h7 77J#8,bhhW:w"7rxxH%o6LAy)nG F*#g-	2; IJ/01q)*}- 1: HWH-./q(7({+ 7 #3
;'5
#$rE   r   c                     | dk  ry| dz
  }t        |dz        }||z  }||z  dz   | k7  r4t        d|dz         D ]!  }||z  dk(  s||z  }||z  dz   | k(  s||fc S  y||fS )Nr   )r   r   g      ?r   )r   r   )r   grid_patchesr   r   rcs         rD   r   z.ColModernVBERT._compute_rows_cols_from_patchesW  s    ! #Q <$%t# $;?k)1lQ./!#q($)A1uqyK/ !t	 0 TzrE   c                 ,    dd| j                   z  z   dz   S )Nz%<fake_token_around_image><global-img><image><fake_token_around_image>)r>   )rB   s    rD   "_create_single_image_prompt_stringz1ColModernVBERT._create_single_image_prompt_stringp  s)    $,,,- **	
rE   r   r   c           	          d}t        |      D ]<  }t        |      D ]'  }|dd|dz    d|dz    dz   d| j                  z  z   z  }) |dz  }> |d	d| j                  z  z   dz   z  }|S )
N r   z<row_r   _col_>r   
z&
<fake_token_around_image><global-img>)r   r>   )rB   r   r   text_split_imagesn_hn_ws         rD   !_create_split_image_prompt_stringz0ColModernVBERT._create_split_image_prompt_stringx  s     ;CT{!/cAgYeC!G9A67$"4"445! # %  	$,,,- **	
 ! rE   c                 (   |dk(  r|dk(  r| j                         }n| j                  ||      }| j                  j                  d|      }| j                  j                  |      }t        j                  |j                  t        j                        S )Nr   r   rb   )
r   r   VISUAL_PROMPT_PREFIXreplacerv   encoderf   arrayr   r   )rB   r   r   image_prompt_tokensexpanded_promptrz   s         rD   r   z)ColModernVBERT._build_input_ids_for_image  s}    19"&"I"I"K"&"H"Ht"T 33;;IGZ[ ..''8 xx28844rE   c                     | j                   j                  J d       |j                  j                  |j                  j                  d   d| j                   j                        S )r_   zModel dim is not definedr   )r6   r   ro   reshapere   rp   s     rD   _post_process_onnx_image_outputz.ColModernVBERT._post_process_onnx_image_output  s_     %%))5Q7QQ5""**%%a("d.D.D.H.H
 	
rE   parallelc              +     K    | j                   d| j                  t        | j                        |||| j                  | j
                  | j                  | j                  | j                  | j                  d|E d{    y7 w)ac  
        Encode a list of documents into list of embeddings.

        Args:
            documents: Iterator of documents or single document to embed
            batch_size: Batch size for encoding -- higher values will use more memory, but be faster
            parallel:
                If > 1, data-parallel encoding will be used, recommended for offline encoding of large datasets.
                If 0, use all available cores.
                If None, don't use data-parallel processing, use default onnxruntime threading instead.

        Returns:
            List of embeddings, one per document
        )r%   r&   rr   ri   r   r(   r)   r*   r0   r-   rM   N )
_embed_documentsr%   r7   r&   r(   r)   r*   r:   r8   r4   )rB   rr   ri   r   r.   s        rD   
embed_textzColModernVBERT.embed_text  s}     * )4(( 
$..)!nn!33 $ 9 9"&"="=
 
 	
 	
   A;B=B>Bc              +     K    | j                   d| j                  t        | j                        |||| j                  | j
                  | j                  | j                  | j                  | j                  d|E d{    y7 w)aa  
        Encode a list of images into list of embeddings.

        Args:
            images: Iterator of image paths or single image path to embed
            batch_size: Batch size for encoding -- higher values will use more memory, but be faster
            parallel:
                If > 1, data-parallel encoding will be used, recommended for offline encoding of large datasets.
                If 0, use all available cores.
                If None, don't use data-parallel processing, use default onnxruntime threading instead.

        Returns:
            List of embeddings, one per document
        )r%   r&   r   ri   r   r(   r)   r*   r0   r-   rM   Nr   )
_embed_imagesr%   r7   r&   r(   r)   r*   r:   r8   r4   )rB   r   ri   r   r.   s        rD   embed_imagezColModernVBERT.embed_image  s}     * &4%% 
$..)!nn!33 $ 9 9"&"="=
 
 	
 	
r   c                     t         S r   )!ColModernVBERTTextEmbeddingWorkerrH   s    rD   _get_text_worker_classz%ColModernVBERT._get_text_worker_class  s    00rE   c                     t         S r   )"ColModernVBERTImageEmbeddingWorkerrH   s    rD   _get_image_worker_classz&ColModernVBERT._get_image_worker_class  s    11rE   )NNNFNFNN)rF   N)i   F)   N)   N)1__name__
__module____qualname____doc__r   ru   r7   r   r   r   r   boollistr   r2   classmethodr   rJ   rA   dictr   rl   r   r   rq   r   r{   r   r
   r   staticmethodtupler   rf   ndarrayr   r   r   r   r   r   r   r   r   r   r   r   r   __classcell__)rC   s   @rD   r$   r$   %   s    	Y   4
 $(!%6:*.#'-1B#B# C=B# #	B#
 H\23B# B# T#Y'B# B# C=B# &c]B# B#H /t,A'B / /C>sJ/;>	c:o	(#!# 
*	#$s) s tH~  "'	Xc]"   	
  
"
tJ'7 
3 
K\ 
, %1Z()%1	z:tCH~5	6%1 %1N?sBJJ/?;>?	c:o	?B S U38_  0
C 
!c ! ! !.5s 5# 5"** 5"
!
 
*	
* "&	"
#&"
 "
 3-	"

 "
 
*	"
N "&	"
Xj11"
 "
 3-	"

 "
 
*	"
H 1t,?
,K'L 1 1 2-A*-M(N 2 2rE   r$   c                   $    e Zd ZdedededefdZy)r   r%   r&   r.   rF   c                      t        d||dd|S Nr   )r%   r&   r'   r   r$   rB   r%   r&   r.   s       rD   init_embeddingz0ColModernVBERTTextEmbeddingWorker.init_embedding  '     
!
 	
 	
rE   Nr   r   r   r7   r   r$   r   r   rE   rD   r   r     $    
 
 
 
P^ 
rE   r   c                   $    e Zd ZdedededefdZy)r   r%   r&   r.   rF   c                      t        d||dd|S r   r   r   s       rD   r   z1ColModernVBERTImageEmbeddingWorker.init_embedding  r   rE   Nr   r   rE   rD   r   r     r  rE   r   )'r   typingr   r   r   r   r   rS   numpyrf   
tokenizersr   PILr	   fastembed.commonr
   "fastembed.common.model_descriptionr   r   fastembed.common.onnx_modelr   fastembed.common.typesr   r   fastembed.common.utilsr   r   Pfastembed.late_interaction_multimodal.late_interaction_multimodal_embedding_baser   ;fastembed.late_interaction_multimodal.onnx_multimodal_modelr   r   r   r"   r   __annotations__r$   r   r   r   rE   rD   <module>r     s     : :     ' Q 9 ; ?  %_6712	@ &;!< [2;=PQ[=\ [2|
(;J(G 

)=j)I 
rE   