
    $HJj                         d dl Z d dlZddlmZmZmZ ddlmZ ddlm	Z	 ddl
mZ ddlmZ ddlmZ dd	lmZ dd
lmZmZ ddlmZ ddlmZ ddlmZmZmZmZ  G d de      Zy)    N   )combine_passagessplit_long_passagessplit_text_passages)
get_piecesnormalize_letter_vinormalize_currency_vispell_acronyms_vinormalize_number_vinormalize_measurement_vinormalize_datenormalize_timenormalize_roman_numbersnormalize_abbreviations_vi)DEFAULT_PIECE_MAX_LENGTHDEFAULT_SENTENCE_MAX_LENGTH%opening_brackets_and_punctutations_repunctutations_rec                       e Zd ZddZd Zd Zd Zd Zd Zd Z	d Z
d	 Zd
 Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd ZdefdZdefdZy)	ViCleanerc                      || _         || _        y N)
clean_abbrclean_acronym)selfr!   r"   s      c/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/misaki/vi_cleaner/cleaner.py__init__zViCleaner.__init__   s    $*    c                 &    |j                  dd      S )N
 )replacer#   texts     r$   
join_lineszViCleaner.join_lines   s    ||D#&&r&   c                 .    |j                  d      \  }}|S Nr   groups)r#   matchspacepunctuations       r$   '_collapse_whitespace_before_punctuationz1ViCleaner._collapse_whitespace_before_punctuation   s    "\\!_{r&   c                 4    |j                  d      \  }}||z   S r/   r0   )r#   r2   r4   r3   s       r$   "_collapse_whitespace_after_bracketz,ViCleaner._collapse_whitespace_after_bracket    s    "\\!_U{""r&   c                 D   t        j                  dd|      }t        j                  t        | j                  |      }t        j                  t        | j
                  |      }t        j                  dd|      }t        j                  dd|      }|j                         }|S )Nz
(\s)\1{1,}z\1z\t+r)   )resubr   r5   r   r7   stripr+   s     r$   collapse_whitespacezViCleaner.collapse_whitespace$   s    vvmUD1vv&BBDJvv;==tEvvmUD1vvfc4(zz|r&   c                 "    |j                         S r    )lowerr+   s     r$   	lowercasezViCleaner.lowercase/   s    zz|r&   c                 8    | j                  |      }d|z   dz   }|S Nr)   )r<   r+   s     r$   clean_basiczViCleaner.clean_basic2   s%    ''-TzCr&   c                 .    t        j                  d|      S )NNFC)unicodedata	normalizer+   s     r$   normalize_ascii_vizViCleaner.normalize_ascii_vi7   s    $$UD11r&   c                     t        |      }|S r    r   r+   s     r$   expand_abbreviationszViCleaner.expand_abbreviations:   s    )$/r&   c                     t        |      S r    r   r+   s     r$   expand_acronymszViCleaner.expand_acronyms>   s     &&r&   c                     t        |      S r    r   r+   s     r$   expand_roman_numberszViCleaner.expand_roman_numbersA   s    &t,,r&   c                 2    t        |      }t        |      }|S r    r   r+   s     r$   expand_date_timezViCleaner.expand_date_timeD   s    d#d#r&   c                 P    t        j                  dd|t         j                        S )Nu   tháng bốnu
   tháng tư)flags)r9   r:   
IGNORECASEr+   s     r$   change_thang_bon_to_thang_tuz&ViCleaner.change_thang_bon_to_thang_tuI   s    vvnlDNNr&   c                     t        |      S r    r   r+   s     r$   expand_measurement_unitsz"ViCleaner.expand_measurement_unitsL   s    '--r&   c                     t        |      S r    r
   r+   s     r$   expand_currencyzViCleaner.expand_currencyO   s    $T**r&   c                     t        |      S r    r   r+   s     r$   expand_numberzViCleaner.expand_numberR       "4((r&   c                     t        |      S r    r   r+   s     r$   expand_letterzViCleaner.expand_letterU   rZ   r&   c                 4    t        j                  dd|      }|S )Nz(/)u    trên r9   r:   r+   s     r$   normalize_splashzViCleaner.normalize_splashX   s    vvfi.r&   c                 0    t        j                  dd|      S )Nz([^\s])(-)([^\s])z\1 \3r^   r+   s     r$   remove_left_hyphenzViCleaner.remove_left_hyphen\   s    vv*Hd;;r&   c                 p    t        j                  d|      D cg c]  }|j                          c}S c c}w )N[\n]+)r9   splitr;   )r#   r,   es      r$   normalize_linebreakzViCleaner.normalize_linebreak_   s-    #%88Hd#;<#;a	#;<<<s   3c                 D   | j                  |      }d|z   dz   }| j                  |      }| j                  r| j                  |      }| j	                  |      }| j
                  r| j                  |      }| j                  |      }| j                  |      }| j                  |      }| j                  |      }| j                  |      }| j                  |      }| j                  |      }| j                  |      }| j                  |      }| j                  |      }|S rA   )r<   rG   r!   rI   rM   r"   rK   rO   rU   rW   rY   r\   r_   ra   rS   r?   r+   s     r$   
clean_textzViCleaner.clean_textb   s   ''-TzC&&t,??,,T2D((.''-D$$T*,,T2##D)!!$'!!$'$$T*&&t,''-006~~d#r&   Nc                 6   ||n| j                   }t        j                  dd|      }| j                  |      }g }g }|D ]F  }t	        ||      }||z  }t        |      dkD  r||d   t        |      z   gz  }8|t        |      gz  }H ||dt        |      dz
   fS )Nz(?<![.!?])[
]+z.
r   r   )r,   r9   r:   rf   r   len)r#   r,   	maxLengthpassagesresultbreakspassagetemps           r$   split_sentenceszViCleaner.split_sentencesy   s    (ttyyvv'5++D1Ggy1DdNF6{Q6":c$i/003t9+%   vaFA...r&   c           	         ||n| j                   }t        |ddd      }t        ||      }|D cg c]  }t        ||       }}|D cg c]  }t	        |       }}t        t	        |            D cg c]  }t        |d |dz           }}g }	|D ]  }
|
D ]  }|	j                  |         t        |	|      }||fS c c}w c c}w c c}w )Nrc   Fz
	 r   )r,   r   r   r   rk   rangesumappend)r#   r,   rl   rm   sub_passagesicombined_sub_passagessub_passages_lensro   	flat_listsublistitemrn   s                r$   split_passageszViCleaner.split_passages   s    (ttyy&tXugF*8Y? $0!1#/a "2y"#/ 	 !1-BC-BSV-BC %6!78:8a '1-.8 	 :	,G  &   - ")Y7v~!1C:s   B=C2C)TT)__name__
__module____qualname__r%   r-   r5   r7   r<   r?   rB   rG   rI   rK   rM   rO   rS   rU   rW   rY   r\   r_   ra   rf   rh   r   rr   r   r~    r&   r$   r   r      s    
'#	
2'-
O.+))<=. $(3K / #'2M r&   r   ) r9   rE   passage_utilsr   r   r   sentence_utilsr   	letter_vir	   currency_vir   
acronym_vir   numberical_vir   measurement_vir   datestime_vir   r   roman_number_vir   abbreviation_vir   	symbol_vir   r   r   r   objectr   r   r&   r$   <module>r      sF    	  U U & * . ) . 4 8 4 7 F  FE Er&   