o nkj*7@s8UddlmZddlZddlZddlZddlmZddlmZddl m Z ddl m Z ddl mZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%dsd dZ&dtddZ'duddZ(e)dde*DZ+de,d<dde+DZ-de,d<dvddZ.dtdd Z/dtd!d"Z0dtd#d$Z1dtd%d&Z2dtd'd(Z3dtd)d*Z4dtd+d,Z5dtd-d.Z6dtd/d0Z7dtd1d2Z8dtd3d4Z9dtd5d6Z:dtd7d8Z;dtd9d:Zd?Z> @dxdydDdEZ?e ddFdzdHdIZ@d{dKdLZAd|dNdOZBd}d~dSdTZCddXdYZDddZd[ZEd\ejFd]fddadbZG  cdddqdrZHdS)) annotationsN) bisect_right)IncrementalDecoder) lru_cache) Generator)ENCODING_MARKSIANA_SUPPORTED_SIMILARRE_POSSIBLE_ENCODING_INDICATIONUNICODE_RANGES_COMBINED_SECONDARY_RANGE_NAMESCOMMON_CJK_CHARACTERS_LATIN_CJK_HANGUL _KATAKANA _HIRAGANA_HALFWIDTH_KATAKANA_THAI_ARABIC_ARABIC_ISOLATED_FORM _LIGATURE _SUPERSCRIPT_SENTENCE_OPEN_PUNCTUATION_ACCENT_KEYWORDS _ACCENTUATED_KNOWN_MB_DECODERS_KNOWN_MB_CLASSES _IANA_NAMES_MULTIBYTE_SEARCH_RADIUS characterstrreturnintcCszt|}Wn tyYdSwd}d|vr|tO}d|vr$|tO}d|vr,|tO}d|vr<|tO}d|vr<|tO}d|vrD|tO}d|vrL|t O}d |vr\|t O}d |vr\|t O}d |vse| d ri|t O}d |vrq|tO}|dvry|tO}tD] }||vr|tO}|Sq{|S)zRCompute all name-based classification flags with a single unicodedata.name() call.rLATINCJKHANGULKATAKANA HALFWIDTHHIRAGANATHAIARABICz ISOLATED FORMLIGATUREz LETTER AE SUPERSCRIPT>INVERTED QUESTION MARKINVERTED EXCLAMATION MARK) unicodedataname ValueErrorrrrrrrrrrendswithrrrrr)r descflagskwr7V/home/djax/ivt_ai_plugin/venv/lib/python3.10/site-packages/charset_normalizer/utils.py_character_flags'sJ r9boolcCtt|t@SN)r:r9rr r7r7r8is_accentuatedQr>cCs.t|}|s |S|d}tt|ddS)N r)r0 decompositionsplitchrr#)r decomposedcodesr7r7r8 remove_accentUs  rGccs"|] \}}|j|j|fVqdSr<)startstop).0r1 ord_ranger7r7r8 as  rLzlist[tuple[int, int, str]]_UNICODE_RANGES_SORTEDcCsg|]}|dqS)rr7)rJer7r7r8 esrOz list[int]_UNICODE_RANGE_STARTS str | NonecCsTt|}|dkr dS|dkrdStt|d}|dkr(t|\}}}||kr(|SdS)zK Retrieve the Unicode range official name from a single character. zControl characterz Basic LatinrrN)ordrrPrM)r character_ordidxrHrIr1r7r7r8 unicode_rangehsrWcCr;r<)r:r9rr=r7r7r8is_latin}r?rXcCs2t|}d|vr dSt|}|durdSd|vS)NPTF Punctuationr0categoryrWr character_categorycharacter_ranger7r7r8is_punctuations r`cCsBt|}d|vs d|vrdSt|}|durdSd|vo |dkS)NSNTFFormsLor[r]r7r7r8 is_symbols recCs$t|}|dur dSd|vpd|vS)NF Emoticons Pictographs)rW)r r_r7r7r8 is_emoticonsrhcCs.|s|dvr dSt|}d|vp|dvS)N>｜+<>TZ>PcPdPo)isspacer0r\)r r^r7r7r8 is_separators rrcCs||kSr<)islowerisupperr=r7r7r8is_case_variabler?rucCr;r<)r:r9rr=r7r7r8is_cjkr?rvcCr;r<)r:r9rr=r7r7r8 is_hiraganar?rwcCr;r<)r:r9rr=r7r7r8 is_katakanar?rxcCr;r<)r:r9rr=r7r7r8 is_hangulr?rycCr;r<)r:r9rr=r7r7r8is_thair?rzcCr;r<)r:r9rr=r7r7r8 is_arabicr?r{cCr;r<)r:r9rr=r7r7r8is_arabic_isolated_formr?r|cCs|tvSr<)r r=r7r7r8is_cjk_uncommonr} range_namecCs|tvSr<)r )rr7r7r8is_unicode_range_secondaryr~rcCs$| o| o|dko|dkS)Nu)rq isprintabler=r7r7r8is_unprintables r sequencebytes | bytearray search_zonec Cst|ttfs tt|}|dt||}|}d|vr$d|vr$dS|jddd}t |D]}| d dd }t |}|durJ|Sq0dS) zW Extract using ASCII-only decoder any specified encoding in the first n-bytes. Nscodingscharsetasciiignoreerrorsr-_) isinstancebytes bytearray TypeErrorlenminlowerdecoder finditergroupreplacerget) rrseq_len search_bytes lowered_bytes decoded_zonematchspecified_encoding encoding_ianar7r7r8any_specified_encodings r)maxsizer1c CsJ|tvrdStD]}z t||WdStttfy"YqwdS)zQ Verify is a specific encoding is a multi byte one based on it IANA name TF)rr importlib import_modulegetcodec ImportErrorAttributeError LookupError)r1providerr7r7r8is_multi_byte_encodings rtuple[str | None, bytes]cCsJtD] }t|}t|tr|g}|D]}||r!||fSqqdS)z9 Identify and extract SIG/BOM in given sequence. )N)rrr startswith)r iana_encodingmarksmarkr7r7r8identify_sig_or_boms  rrcCs|dvS)N>utf_16utf_32r7)rr7r7r8should_strip_sig_or_bom(r~rTcp_namestrictcCs>|dd}t|}|dur|S|rtd|d|S)zIReturns the Python normalized encoding name (Not the IANA official name).rrNzUnable to retrieve IANA for '')rrrrr2)rrrr7r7r8 iana_name,s r iana_name_a iana_name_bfloatc Cst|st|r dStd|j}td|j}|dd}|dd}d}tdD]}t|g}||||krA|d7}q,|dS)Ngz encodings.rrrr)rrrrrangerr) rr decoder_a decoder_bid_aid_bcharacter_match_counti to_be_decodedr7r7r8 cp_similarity:s    rcCs|tvo |t|vS)z Determine if two code page are at least 80% similar. IANA_SUPPORTED_SIMILAR dict was generated using the function cp_similarity. )r )rrr7r7r8 is_cp_similarNs rcharset_normalizerz)%(asctime)s | %(levelname)s | %(message)slevel format_stringNonecCs:t|}||t}|t|||dSr<)logging getLoggersetLevel StreamHandler setFormatter Formatter addHandler)r1rrloggerhandlerr7r7r8set_logging_handlerYs  rF sequencesroffsetsr chunk_sizebom_or_sig_availablestrip_sig_or_bom sig_payloadris_multi_byte_decoderdecoded_payloaddeferred_decodingGenerator[str, None, None]c cs$|r-|dr-t|} t|} |D]} | | | } || | |}|s'dS|VqdS|rH|sH|D]} || | |}|sBdS|Vq3dS| rp|sN|n|t|d}|D]} || | |}|sgdSt||VqXdS|D]} | |}|t|dkrqr|| | |}|r|s||}|j||rdndd}|r | dkr t|d}|d|}d}|rt|} | | t|}td|t}t| |t}||||dk}|r |s ||vr t| | d d D]"}|||}|r|s||}|j|dd}|d||vr nq|VqrdS) Niso2022_rrrrrAF) rrr!rrmaxrfindr)rrrrrrrrrrdecoded_lengthsequence_lengthrdecoded_offsetchunk base_bytes cut_sequence chunk_endchunk_partial_size_chk chunk_prefix found_nearbyexpected_offset search_start search_endjr7r7r8cut_sequence_chunksfs        r)r r!r"r#)r r!r"r:)r r!r"r!)r r!r"rQ)rr!r"r:)r)rrrr#r"rQ)r1r!r"r:)rrr"r)rr!r"r:)T)rr!rr:r"r!)rr!rr!r"r)rr!rr!r"r:)r1r!rr#rr!r"r)NF)rrrr!rrrr#rr:rr:rrrr:rrQrr:r"r)I __future__rrrr0bisectrcodecsr functoolsrtypingrconstantrr r r r r rrrrrrrrrrrrrrrrrrr9r>rGsorteditemsrM__annotations__rPrWrXr`rerhrrrurvrwrxryrzr{r|r}rrrrrrrrrINFOrrr7r7r7r8sb    h  *