o
    nkj                     @  s  U d dl mZ d dlmZ d dlmZ ddlmZmZm	Z	m
Z
mZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZ ddlmZmZmZm Z m!Z!m"Z"m#Z# eeB eB eB eB Z$de%d< G d	d
 d
Z&eddd;ddZ'dd e(dD Z)de%d< G dd dZ*G dd de*Z+G dd de*Z,G dd de*Z-G dd  d e*Z.G d!d" d"e*Z/G d#d$ d$e*Z0G d%d& d&e*Z1G d'd( d(e*Z2G d)d* d*e*Z3G d+d, d,e*Z4eddd<d1d2Z5	4d=d>d9d:Z6dS )?    )annotations)	lru_cache)	getLogger   )COMMON_CJK_CHARACTERSCOMMON_SAFE_ASCII_CHARACTERSTRACECompatibleFamillyRange_ACCENTUATED_ARABIC_ARABIC_ISOLATED_FORM&_BASIC_LATIN_COMPATIBLE_RANGE_FAMILIES_CJK_COMPATIBLE_RANGE_FAMILIES#_COMPATIBLE_WITH_ANY_RANGE_FAMILIES_HANGUL_HALFWIDTH_KATAKANA	_HIRAGANA	_KATAKANA_LATIN	_LIGATURE_RANGE_FAMILIES_SENTENCE_OPEN_PUNCTUATION_SUPERSCRIPT_THAI)_character_flagsis_emoticonis_punctuationis_separator	is_symbolremove_accentunicode_rangeint_GLYPH_MASKc                   @  s   e Zd ZU dZdZded< ded< ded< ded< ded	< ded
< ded< ded< ded< ded< ded< ded< ded< ded< ded< ded< ded< ded< ded< ded< ded< ded< ded< ded< ded< ded < ded!< ded"< d(d%d&Zd'S ))CharInfoz>Pre-computed character properties shared across all detectors.)	character	printablealphaupperlowerspacedigitis_asciicase_variableflagsaccentuatedlatinis_cjkis_katakanais_halfwidth_katakana	is_arabicis_ligatureis_superscriptis_sentence_open_punctuationis_glyphpunctsymrangesepemoticonsafe
common_cjk
unaccentedstrr%   boolr&   r'   r(   r)   r*   r+   r,   r-   r"   r.   r/   r0   r1   r2   r3   r4   r5   r6   r7   r8   r9   r:   
str | Noner;   r<   r=   r>   r?   r@   returnNonec                 C  s  || _ t|}|dk r-d| _d| _|| _d| _d| _|tv | _d| _	d| _
d| _d| _d| _d| _d| _d| _d|  krBdkrgn n#d| _d| _d| _d| _d| _d| _d| _t| _d| _d| _d| _nzd|  krqdkrn n#d| _d| _d| _d| _d| _d| _d| _t| _d| _d| _d| _nKd|  krd	krn n#d| _d| _d| _d| _d| _d| _d| _d
| _d| _d| _d| _n|dksd|  krdkrn n$d| _d| _d| _d| _d| _|dk| _d| _d
| _d| _d| _d| _n| | _d| _d| _d| _d| _d| _d| _d
| _d| _| jrt|nd| _| jr)t|nd| _nd| _d| _| | _|  | _|! | _|" | _|# | _|$ | _| j| jk| _t%|}| jrdd| _nt&|| _|| _t'|t(@ | _t'|t@ | _t'|t)@ | _	t'|t*@ | _
t'|t+@ | _t'|t,@ | _t'|t-@ | _t'|t.@ | _t'|t/@ | _t'|t0@ | _| jr| jrt1|| _n|| _| j	o|t2v | _| jrt|| _t|| _nd| _d| _t3|| _4t5|| _6dS )zkCompute all properties for *character* (built once per codepoint,
        every branch assigns every slot).   TFA   Z   a   z   0   9   r       	      N)7r%   ordr,   r/   r@   r=   r?   r   r>   r1   r2   r3   r4   r5   r6   r7   r8   r'   r(   r)   r*   r+   r&   r-   r   r.   r0   r9   r:   isprintabler   r   isalphaisupperislowerisspaceisdigitr   r   rB   r
   r   r   r   r   r   r   r   r#   r    r   r!   r;   r   r<   )selfr%   or.    rY   S/home/djax/ivt_ai_plugin/venv/lib/python3.10/site-packages/charset_normalizer/md.py__init__k   s   




 










zCharInfo.__init__N)r%   rA   rD   rE   )__name__
__module____qualname____doc__	__slots____annotations__r[   rY   rY   rY   rZ   r$   ,   s@   
 r$   N)maxsizer%   rA   rD   c                 C  s   t | S )zBBuild (once per codepoint) and cache the CharInfo for *character*.)r$   )r%   rY   rY   rZ   
_char_info   s   rc   c                 C  s   g | ]}t t|qS rY   )r$   chr).0
_codepointrY   rY   rZ   
<listcomp>   s    rg   rF   zlist[CharInfo]_ASCII_CHAR_INFOc                   @  s6   e Zd ZdZdZdd	d
ZdddZedddZdS )MessDetectorPluginzy
    Base abstract class used for mess detection plugins.
    All detectors MUST extend and implement given methods.
    rY   r%   rA   infor$   rD   rE   c                 C     t )z
        The main routine to be executed upon character.
        Insert the logic in witch the text would be considered chaotic.
        NotImplementedErrorrW   r%   rj   rY   rY   rZ   	feed_info  s   zMessDetectorPlugin.feed_infoc                 C  rk   )zB
        Permit to reset the plugin to the initial state.
        rl   rW   rY   rY   rZ   reset  s   zMessDetectorPlugin.resetfloatc                 C  rk   )z
        Compute the chaos ratio based on what your feed() has seen.
        Must NOT be lower than 0.; No restriction gt 0.
        rl   rp   rY   rY   rZ   ratio  s   zMessDetectorPlugin.ratioNr%   rA   rj   r$   rD   rE   rD   rE   rD   rr   )	r\   r]   r^   r_   r`   ro   rq   propertyrs   rY   rY   rY   rZ   ri     s    

ri   c                   @  <   e Zd ZdZdddZdd
dZdddZedddZdS ) TooManySymbolOrPunctuationPlugin_punctuation_count_symbol_count_character_count_last_printable_charrD   rE   c                 C  s   d| _ d| _d| _d | _d S Nr   rz   rp   rY   rY   rZ   r[   *  s   
z)TooManySymbolOrPunctuationPlugin.__init__r%   rA   rj   r$   c                 C  s^   |  j d7  _ || jkr*|js*|jr|  jd7  _n|js*|jr*|js*|  jd7  _|| _dS )1Optimized feed using pre-computed character info.r      N)	r}   r~   r>   r9   r{   r+   r:   r=   r|   rn   rY   rY   rZ   ro   1  s   
z*TooManySymbolOrPunctuationPlugin.feed_infoc                 C  s   d| _ d| _d| _d S r   )r{   r}   r|   rp   rY   rY   rZ   rq   =     
z&TooManySymbolOrPunctuationPlugin.resetrr   c                 C  s0   | j dkrdS | j| j | j  }|dkr|S dS )Nr           333333?)r}   r{   r|   )rW   ratio_of_punctuationrY   rY   rZ   rs   B  s   

z&TooManySymbolOrPunctuationPlugin.ratioNru   rt   rv   	r\   r]   r^   r`   r[   ro   rq   rw   rs   rY   rY   rY   rZ   ry   "  s    


ry   c                   @  rx   )TooManyAccentuatedPluginr}   _accentuated_countrD   rE   c                 C     d| _ d| _d S r   r   rp   rY   rY   rZ   r[   Q     
z!TooManyAccentuatedPlugin.__init__r%   rA   rj   r$   c                 C  s*   |  j d7  _ |jr|  jd7  _dS dS r   r   N)r}   r/   r   rn   rY   rY   rZ   ro   U     z"TooManyAccentuatedPlugin.feed_infoc                 C  r   r   r   rp   rY   rY   rZ   rq   \  r   zTooManyAccentuatedPlugin.resetrr   c                 C  s*   | j dk rdS | j| j  }|dkr|S dS )N   r   gffffff?r   )rW   ratio_of_accentuationrY   rY   rZ   rs   `  s   
zTooManyAccentuatedPlugin.ratioNru   rt   rv   r   rY   rY   rY   rZ   r   N  s    


r   c                   @  rx   )UnprintablePlugin_unprintable_countr}   _has_escaperD   rE   c                 C  s   d| _ d| _d| _d S Nr   Fr   rp   rY   rY   rZ   r[   l  r   zUnprintablePlugin.__init__r%   rA   rj   r$   c                 C  sJ   |dkrd| _ |js|js|dkr|dkr|  jd7  _|  jd7  _dS )r   Tu   ﻿r   N)r   r&   r*   r   r}   rn   rY   rY   rZ   ro   q  s   zUnprintablePlugin.feed_infoc                 C  s   d| _ d| _d S r   )r   r   rp   rY   rY   rZ   rq     r   zUnprintablePlugin.resetrr   c                 C  s(   | j dkrdS | jrdS | jd | j  S )Nr   r         ?r   )r}   r   r   rp   rY   rY   rZ   rs     s
   
zUnprintablePlugin.ratioNru   rt   rv   r   rY   rY   rY   rZ   r   i  s    


r   c                   @  rx   )SuspiciousDuplicateAccentPlugin_successive_countr}   _last_latin_character_last_was_accentuatedrD   rE   c                 C     d| _ d| _d | _d| _d S r   r   rp   rY   rY   rZ   r[     s   
z(SuspiciousDuplicateAccentPlugin.__init__r%   rA   rj   r$   c                 C  sn   |  j d7  _ | jdur.|jr.| jr.|jr | jjr |  jd7  _|j| jjkr.|  jd7  _|| _|j| _dS r   )r}   r   r/   r   r(   r   r@   rn   rY   rY   rZ   ro     s   
z)SuspiciousDuplicateAccentPlugin.feed_infoc                 C  r   r   r   rp   rY   rY   rZ   rq        
z%SuspiciousDuplicateAccentPlugin.resetrr   c                 C  s   | j dkrdS | jd | j  S )Nr   r   r   )r}   r   rp   rY   rY   rZ   rs     s   
z%SuspiciousDuplicateAccentPlugin.ratioNru   rt   rv   r   rY   rY   rY   rZ   r     s    


r   c                   @  rx   )SuspiciousRange"_suspicious_successive_range_countr}   _last_printable_seen_last_printable_rangerD   rE   c                 C     d| _ d| _d | _d | _d S r   r   rp   rY   rY   rZ   r[     r   zSuspiciousRange.__init__r%   rA   rj   r$   c                 C  s   |  j d7  _ |js|js|jrd| _d| _dS | jdu r&|| _|j| _dS | j}|j}||ks4|du r@t||r@|  jd7  _|| _|| _dS r   )	r}   r*   r9   r>   r   r   r;    is_suspiciously_successive_ranger   )rW   r%   rj   unicode_range_aunicode_range_brY   rY   rZ   ro     s    


zSuspiciousRange.feed_infoc                 C  r   r   )r}   r   r   r   rp   rY   rY   rZ   rq     r   zSuspiciousRange.resetrr   c                 C  s"   | j dkrdS | jd | j  }|S )NrO   r   r   )r}   r   )rW   ratio_of_suspicious_range_usagerY   rY   rZ   rs     s   
zSuspiciousRange.ratioNru   rt   rv   r   rY   rY   rY   rZ   r     s    


r   c                   @  rx   )SuperWeirdWordPlugin_word_count_bad_word_count_foreign_long_count_is_current_word_bad_foreign_long_watchr}   _bad_character_count_buffer_length_buffer_last_char_buffer_last_char_accentuated_buffer_accent_count_buffer_glyph_count_buffer_upper_count_buffer_first_lower_buffer_has_non_ascii_buffer_last_char_ligature_buffer_has_internal_ligature_is_current_word_invalid_invalid_word_countrD   rE   c                 C  sv   d| _ d| _d| _d| _d| _d| _d| _d| _d | _d| _	d| _
d| _d| _d| _d| _d| _d| _d| _d| _d S r   r   rp   rY   rY   rZ   r[   
  s&   
zSuperWeirdWordPlugin.__init__r%   rA   rj   r$   c                 C  s  |j rX| jr	d| _|j| _| jdkr|j| _|  jd7  _|| _|jr*|  j	d7  _	|j
s0d| _|j| _|jr>|  jd7  _|jrJ|  jd7  _dS | jsV|jrS|jrVd| _dS | js]dS |jsf|jrl| jrld| _d| _|jsv|jsv|jr:|  jd7  _| j}|  j|7  _|dkr| j| dkrd| _n?| jr| j r| j	|kr|  jd7  _d| _n'| jdkrd| _|  jd7  _n| jr| jr| j	|d kr|  jd7  _d| _|dkr| jr| j	dko| j	| dk}|s|  jd7  _d| _| jr	|  jd7  _|  j|7  _d	| _| jr|  j d7  _ d	| _d	| _d| _d| _d	| _d| _d| _d| _	d	| _d	| _d	| _d	| _dS |d
vrY|j!s[|j"r]d| _|  jd7  _|| _d	| _dS dS dS dS )r   Tr   r   N         ?   r   F>   _-<=>|~)#r'   r   r   r5   r   r)   r   r   r(   r   r,   r   r/   r   r   r8   r   r   r0   r7   r6   r   r   r*   r9   r<   r   r}   rS   r   r   r   r   r+   r:   )rW   r%   rj   buffer_lengthprobable_camel_casedrY   rY   rZ   ro   "  s   






zSuperWeirdWordPlugin.feed_infoc                 C  sv   d| _ d | _d| _d| _d| _d| _d| _d| _d| _d| _	d| _
d| _d| _d| _d| _d| _d| _d| _d| _d S r   )r   r   r   r   r   r   r   r}   r   r   r   r   r   r   r   r   r   r   r   rp   rY   rY   rZ   rq     s&   
zSuperWeirdWordPlugin.resetrr   c                 C  s.   | j rdS | jdkr| jdkrdS | j| j S )Nr   
   r   r   )r   r   r   r   r}   rp   rY   rY   rZ   rs     s
   zSuperWeirdWordPlugin.ratioNru   rt   rv   r   rY   rY   rY   rZ   r     s    


dr   c                   @  @   e Zd ZdZdZdddZdddZdddZedddZ	dS )CjkUncommonPluginz<
    Detect messy CJK text that probably means nothing.
    r}   _uncommon_countrD   rE   c                 C  r   r   r   rp   rY   rY   rZ   r[     r   zCjkUncommonPlugin.__init__r%   rA   rj   r$   c                 C  s*   |  j d7  _ |js|  jd7  _dS dS r   )r}   r?   r   rn   rY   rY   rZ   ro     r   zCjkUncommonPlugin.feed_infoc                 C  r   r   r   rp   rY   rY   rZ   rq     r   zCjkUncommonPlugin.resetrr   c                 C  s.   | j dk rdS | j| j  }|dkr|d S dS )Nr   r   r      r   )rW   uncommon_form_usagerY   rY   rZ   rs     s   
zCjkUncommonPlugin.ratioNru   rt   rv   
r\   r]   r^   r_   r`   r[   ro   rq   rw   rs   rY   rY   rY   rZ   r     s    


r   c                   @  r   )SuspiciousKatakanaPluginzDDetect implausible halfwidth Katakana and uncommon CJK combinations._katakana_count_halfwidth_katakana_count
_cjk_count_uncommon_cjk_countrD   rE   c                 C     d| _ d| _d| _d| _d S r   r   rp   rY   rY   rZ   r[     r   z!SuspiciousKatakanaPlugin.__init__r%   rA   rj   r$   c                 C  sV   |j r|  jd7  _|jr|  jd7  _dS |  jd7  _|js)|  jd7  _dS dS r   )r2   r   r3   r   r   r?   r   rn   rY   rY   rZ   ro     s   z"SuspiciousKatakanaPlugin.feed_infoc                 C  r   r   r   rp   rY   rY   rZ   rq     r   zSuspiciousKatakanaPlugin.resetrr   c                 C  s:   | j dkr| j | jkrd| j  kr| jkrdS  dS dS )Nr      r   r   )r   r   r   r   rp   rY   rY   rZ   rs     s   
zSuspiciousKatakanaPlugin.ratioNru   rt   rv   r   rY   rY   rY   rZ   r     s    


r   c                   @  rx   )ArchaicUpperLowerPlugin_buf_character_count_since_last_sep_successive_upper_lower_count#_successive_upper_lower_count_finalr}   _last_alpha_seen_upper_last_alpha_seen_lower_current_ascii_onlyrD   rE   c                 C  s4   d| _ d| _d| _d| _d| _d| _d| _d| _d S )NFr   Tr   rp   rY   rY   rZ   r[     s   
z ArchaicUpperLowerPlugin.__init__r%   rA   rj   r$   c                 C  s   |j o|j}| }|r8| jdkr8| jdkr#|js#| js#|  j| j7  _d| _d| _d| _|  jd7  _d| _dS | jrA|j	sAd| _| jdkrg|j
rL| jsR|jrd| jrd| jr`|  jd7  _d| _nd| _nd| _|  jd7  _|  jd7  _|j
| _|j| _dS )r   r   @   Fr   TNr   )r'   r-   r   r+   r   r   r   r   r}   r,   r(   r   r)   r   )rW   r%   rj   is_concerned	chunk_seprY   rY   rZ   ro     sD   

z!ArchaicUpperLowerPlugin.feed_infoc                 C  s4   d| _ d| _d| _d| _d| _d| _d| _d| _d S )Nr   FT)r}   r   r   r   r   r   r   r   rp   rY   rY   rZ   rq   ;  s   
zArchaicUpperLowerPlugin.resetrr   c                 C  s   | j dkrdS | j| j  S )Nr   r   )r}   r   rp   rY   rY   rZ   rs   E  s   
zArchaicUpperLowerPlugin.ratioNru   rt   rv   r   rY   rY   rY   rZ   r     s    


+
r   c                   @  s<   e Zd ZdZdddZdddZdddZedddZdS )ArabicIsolatedFormPluginr}   _isolated_form_countrD   rE   c                 C  r   r   r   rp   rY   rY   rZ   r[   P  r   z!ArabicIsolatedFormPlugin.__init__c                 C  r   r   r   rp   rY   rY   rZ   rq   T  r   zArabicIsolatedFormPlugin.resetr%   rA   rj   r$   c                 C  s.   |  j d7  _ |jt@ r|  jd7  _dS dS r   )r}   r.   r   r   rn   rY   rY   rZ   ro   X  s   
z"ArabicIsolatedFormPlugin.feed_inforr   c                 C  s   | j dk rdS | j| j  }|S )Nr   r   r   )rW   isolated_form_usagerY   rY   rZ   rs   _  s   
zArabicIsolatedFormPlugin.ratioNru   rt   rv   )	r\   r]   r^   r`   r[   rq   ro   rw   rs   rY   rY   rY   rZ   r   M  s    


r   r   rC   r   rB   c                 C  sz   | du s|du r
dS t |  }t | }||krdS |tv s |tv r"dS t||tv r+dS | dkr3|tvS |dkr;|tvS dS )z_
    Determine if two Unicode ranges seen next to each other can be considered suspicious.
    NTFzBasic Latin)r   r   r	   r   r   )r   r   	familly_a	familly_brY   rY   rZ   r   i  s    r   皙?Fdecoded_sequencemaximum_thresholdrr   debugc           $   
   C  s  t | }|dk rd}n	|dk rd}nd}|  }t}t}t }t }	t }
t }t }t	 }t
 }t }t }t }|j}|	j}|
j}|j}|j}|j}|j}|j}|j}|j}td||D ]}| |||  D ]h}t|}|dk rw|| }n||}||| ||| |r|jr||| qh||| |jr||| ||| |jr||| |jr||| |jr||| ||| n|jr||| |jr||| qh|j|	j |
j |j |j |j |j |j |j |j } | |kr n5q^|d }!|d|! |s|d|! |d|! |j|	j |
j |j |j |j |j |j |j |j } |r|td	}"|"td
| d|  d|  |dkr_|"td| dd   |"td| dd   ||	|
|||||||f
D ]}#|"t|#j d|#j  qkt| dS )zw
    Compute a mess ratio given a decoded bytes sequence. The maximum threshold does stop the computation earlier.
    i  rM   i   r   rF   r   r   
charset_normalizerzIMess-detector extended-analysis start. intermediary_mean_mess_ratio_calc=z mean_mess_ratio=z maximum_threshold=   zStarting with: NzEnding with: iz: r   )lenisasciirh   rc   ry   r   r   r   r   r   r   r   r   r   ro   r;   rP   r&   r'   r0   r1   r2   r4   rs   r   logr   	__class__round)$r   r   r   seq_lenstepis_pure_ascii
ascii_info	char_infod_spd_tad_upd_sdad_srd_swd_cud_skd_aud_ai	d_sp_feed	d_ta_feed	d_up_feed
d_sda_feed	d_sr_feed	d_sw_feed	d_cu_feed	d_sk_feed	d_au_feed	d_ai_feedblock_startr%   	codepointrj   mean_mess_rationl_infologgerdtrY   rY   rZ   
mess_ratio  s   	











	


	

r  )r%   rA   rD   r$   )r   rC   r   rC   rD   rB   )r   F)r   rA   r   rr   r   rB   rD   rr   )7
__future__r   	functoolsr   loggingr   constantr   r   r   r	   r
   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   utilsr   r   r   r   r   r    r!   r#   ra   r$   rc   r;   rh   ri   ry   r   r   r   r   r   r   r   r   r   r   r  rY   rY   rY   rZ   <module>   s:    \$ M,%,9 4".W$