Nj4ddlmZddlZddlZddlmZddlmZmZddl Z ddl Z ddl Z ddl mZddlmZddlmZddlmZdd lmZmZdd lmZdd lmZej8eZ d dd Z dd Z ddZ!ddZ"y)) annotationsN)Path)Anycast) save_file) Tokenizer)create_model_card)get_metadata_from_readme)FOLDER_LAYOUTSLayout)maybe_get_cached_model_path)SafeOpenProtocolc |r||z n|}|jddd|i} ||| d<||| d<t| |dz |jt|dz d t |} t j |j j| d <|t|jd | d <n| jd dtj| t|dz ddd ddddg} | jdr| jdddddtj| t|dz ddt j#d||r t%|fi|yy)a Save a model to a folder. :param folder_path: The path to the folder. :param embeddings: The embeddings. :param tokenizer: The tokenizer. :param config: A metadata config. :param create_model_card: Whether to create a model card. :param subfolder: The subfolder to save the model in. :param weights: The weights of the model. If None, no weights are saved. :param mapping: The token mapping of the model. If None, there is no token mapping. :param **kwargs: Any additional arguments. T)exist_okparents embeddingsNweightsmappingzmodel.safetensorsztokenizer.jsonF)prettyembedding_dtypervocabulary_quantizationz config.jsonw)indent0.z,sentence_transformers.models.StaticEmbedding)idxnamepathtype normalize1 1_Normalizez&sentence_transformers.models.Normalizez modules.jsonzSaved model to )mkdirrsavestrdictnpdtyperintshapepopjsondumpopengetappendloggerinfomake_model_card) folder_pathr tokenizerconfigr subfolderrrkwargs model_weightscfgmoduless r/Users/ahmed/devFolder/Ultron/claude-voice/.venv/lib/python3.12/site-packages/model2vec/persistence/persistence.pysave_pretrainedr?sm..7+ )KKtT2!:.M#* i #* i  m[+>>? NN3{%556uNE v,CXXj&6&67<qI KK/+/0 .v.czt|}t|||}|r||z }t|}|dz }ttt j |jd}|jrdnd}|j|} |jd} |jd } |jr t|} ni} tjt|j } t#j$t'|j(}| | || | | fS#t$rd} YwxYw#t$rd} YwxYw) agLoads a pretrained model from a folder. :param folder_or_repo_path: The folder or repo path to load from. - If this is a local path, we will load from the local path. - If the local path is not found, we will attempt to load from the huggingface hub. :param subfolder: The subfolder to load from. :param token: The huggingface token to use. :param force_download: Whether to force the download of the model. If False, the model is only downloaded if it is not already present in the cache. :return: The embeddings, tokenizer, config, metadata, weights and mapping. )folder_or_repo_pathtokenforce_downloadz README.mdnumpy) frameworkzembedding.weightrrNr)r_resolve_folder _get_pathsrr safetensors safe_openris_sentence_transformers get_tensor Exceptionexistsr r from_filer'r7r.loadr0r8)rBr9rCrDfolderselected_layout readme_pathopened_tensor_fileembedding_namerrrmetadatar7r8s r>load_pretrainedrWQs?$231DEbp qF)# (O;&K. 0E0EoF`F`ls0tu+:+S+S'YeN#..~>J$// :$// :+K8$..s?3L3L/MNI YYtO223 4F y&(GW DD!  s$DD, D)(D), D:9D:c|jr|S|stt|x}r|Stt j t|j d|}|S)z2Resolve a folder locally or from hugging face hub.model) repo_typerC)rNr r'rhuggingface_hubsnapshot_downloadas_posix)rBrCrDrQs r>rGrGsh!!#"" 05H1IJ J6 JM ))#.A.J.J.L*MY`hmnF Mr@ctD]'}|j|}|js%|cStd|d)z-Get all paths by trying out multiple layouts.z'Could not find expected model files in zH. Tried model2vec, sentence-transformers, and 0_StaticEmbedding layouts.)r with_parentis_valid ValueError)rQlayouts r>rHrHsM ##F+ ?? M!  1&:Q Q r@)TNNN)r6rrz np.ndarrayr7rr8zdict[str, Any]r boolr9 str | Nonernp.ndarray | Nonerrer:rreturnNone) rBz str | Pathr9rdrCrdrDrcrfzbtuple[np.ndarray, Tokenizer, dict[str, Any], dict[str, Any], np.ndarray | None, np.ndarray | None])rBrrCrdrDrcrfr)rQrrfr )# __future__rr.loggingpathlibrtypingrrr[rEr)rIsafetensors.numpyr tokenizersrmodel2vec.modelcardsr r5r model2vec.persistence.datamodelsr r model2vec.persistence.hfr model2vec.utilsr getLogger__name__r3r?rWrGrHr@r>rus" ' E9C@,   8 $# !%!%7/7/7/7/  7/  7/  7/7/7/7/ 7/t3E#3E3E 3E 3E h 3El" r@