File: //opt/saltstack/salt/lib/python3.10/site-packages/charset_normalizer/__pycache__/md.cpython-310.pyc
o
�N�g�H �
@ sf d dl mZ d dlmZ d dlmZmZ ddlmZm Z m
Z
ddl m Z m
Z
mZmZmZmZmZmZmZmZmZmZmZmZmZmZ G dd� d�Z G d d
� d
e �Z G d d � d e �Z G d
d� de �Z G dd� de �Z G dd� de �Z!G dd� de �Z"G dd� de �Z#G dd� de �Z$edd�dee% d ee% d e&fd d ��Z'ed d� "d)d#e%d$e(d%e&d e(fd&d'��Z)d(S )*� )� lru_cache)� getLogger)�List�Optional� )� COMMON_SAFE_ASCII_CHARACTERS�TRACE� UNICODE_SECONDARY_RANGE_KEYWORD)�is_accentuated�is_ascii�is_case_variable�is_cjk� is_emoticon� is_hangul� is_hiragana� is_katakana�is_latin�is_punctuation� is_separator� is_symbol�is_thai�is_unprintable�
remove_accent�
unicode_rangec @ sP e Zd ZdZdedefdd�Zdeddfdd�Zd
d d
�Ze de
fd d ��Z dS )�MessDetectorPluginzy
Base abstract class used for mess detection plugins.
All detectors MUST extend and implement given methods.
� character�returnc C � t �)z@
Determine if given character should be fed in.
��NotImplementedError��selfr � r" �I/opt/saltstack/salt/lib/python3.10/site-packages/charset_normalizer/md.py�eligible$ � zMessDetectorPlugin.eligibleNc C r )z�
The main routine to be executed upon character.
Insert the logic in witch the text would be considered chaotic.
r r r" r" r# �feed* s zMessDetectorPlugin.feedc C r )zB
Permit to reset the plugin to the initial state.
r �r! r" r" r# �reset1 r% zMessDetectorPlugin.resetc C r )z�
Compute the chaos ratio based on what your feed() has seen.
Must NOT be lower than 0.; No restriction gt 0.
r r' r" r" r# �ratio7 s zMessDetectorPlugin.ratio�r N) �__name__�
__module__� __qualname__�__doc__�str�boolr$ r& r( �property�floatr) r" r" r" r# r s
r c @ �V e Zd Zddd�Zdedefdd�Zdeddfdd �Zdd
d �Ze de
fd d
��Z dS )� TooManySymbolOrPunctuationPluginr Nc C s" d| _ d| _d| _d | _d| _d S )Nr F)�_punctuation_count�
_symbol_count�_character_count�_last_printable_charZ_frenzy_symbol_in_wordr' r" r" r# �__init__A s
z)TooManySymbolOrPunctuationPlugin.__init__r c C � |� � S �N�� isprintabler r" r" r# r$ I � z)TooManySymbolOrPunctuationPlugin.eligiblec C sp | j d7 _ || jkr3|tvr3t|�r | jd7 _n|�� du r3t|�r3t|�du r3| jd7 _|| _d S )Nr F� ) r7 r8 r r r5 �isdigitr r r6 r r" r" r# r& L s
�
z%TooManySymbolOrPunctuationPlugin.feedc C s d| _ d| _d| _d S �Nr )r5 r7 r6 r' r" r" r# r( ^ �
z&TooManySymbolOrPunctuationPlugin.resetc C s0 | j dkrdS | j| j | j }|dkr|S dS )Nr � �333333�?)r7 r5 r6 )r! Zratio_of_punctuationr" r" r# r) c s
�z&TooManySymbolOrPunctuationPlugin.ratior* � r+ r, r- r9 r/ r0 r$ r& r( r1 r2 r) r" r" r" r# r4 @ s
r4 c @ r3 )�TooManyAccentuatedPluginr Nc C � d| _ d| _d S rA �r7 �_accentuated_countr' r" r" r# r9 p �
z!TooManyAccentuatedPlugin.__init__r c C r: r; )�isalphar r" r" r# r$ t r> z!TooManyAccentuatedPlugin.eligiblec C s, | j d7 _ t|�r| jd7 _d S d S �Nr )r7 r
rI r r" r" r# r& w s �z TooManyAccentuatedPlugin.feedc C rG rA rH r' r" r" r# r( } rJ z TooManyAccentuatedPlugin.resetc C s4 | j dks
| j dk r dS | j| j }|dkr|S dS )Nr � rC gffffff�?rH )r! Zratio_of_accentuationr" r" r# r) � s z TooManyAccentuatedPlugin.ratior* rE r" r" r" r# rF o s
rF c @ r3 )�UnprintablePluginr Nc C rG rA )�_unprintable_countr7 r' r" r" r# r9 � rJ zUnprintablePlugin.__init__r c C � dS �NTr" r r" r" r# r$ � � zUnprintablePlugin.eligiblec C s( t |�r | jd7 _| jd7 _d S rL )r rO r7 r r" r" r# r& � s zUnprintablePlugin.feedc C s
d| _ d S rA )rO r' r" r" r# r( � s
zUnprintablePlugin.resetc C � | j dkrdS | jd | j S )Nr rC rM )r7 rO r' r" r" r# r) � �
zUnprintablePlugin.ratior* rE r" r" r" r# rN � s
rN c @ r3 )� SuspiciousDuplicateAccentPluginr Nc C � d| _ d| _d | _d S rA ��_successive_countr7 �_last_latin_characterr' r" r" r# r9 � s
z(SuspiciousDuplicateAccentPlugin.__init__r c C s |� � ot|�S r; )rK r r r" r" r# r$ � s z(SuspiciousDuplicateAccentPlugin.eligiblec C st | j d7 _ | jd ur5t|�r5t| j�r5|�� r%| j�� r%| jd7 _t|�t| j�kr5| jd7 _|| _d S rL )r7 rY r
�isupperrX r r r" r" r# r& � s
��
z$SuspiciousDuplicateAccentPlugin.feedc C rV rA rW r' r" r" r# r( � rB z%SuspiciousDuplicateAccentPlugin.resetc C rS )Nr rC r? )r7 rX r' r" r" r# r) � rT z%SuspiciousDuplicateAccentPlugin.ratior* rE r" r" r" r# rU � s
rU c @ r3 )�SuspiciousRanger Nc C rV rA )�"_suspicious_successive_range_countr7 �_last_printable_seenr' r" r" r# r9 � rB zSuspiciousRange.__init__r c C r: r; r< r r" r" r# r$ � r> zSuspiciousRange.eligiblec C sx | j d7 _ |�� st|�s|tv rd | _d S | jd u r"|| _d S t| j�}t|�}t||�r7| jd7 _|| _d S rL )r7 �isspacer r r] r � is_suspiciously_successive_ranger\ )r! r �unicode_range_a�unicode_range_br" r" r# r& � s ��
zSuspiciousRange.feedc C rV rA )r7 r\ r] r' r" r" r# r( � rB zSuspiciousRange.resetc C s. | j dkrdS | jd | j }|dk rdS |S )Nr rC r? g�������?)r7 r\ )r! Z ratio_of_suspicious_range_usager" r" r# r) � s
�zSuspiciousRange.ratior* rE r" r" r" r# r[ � s
r[ c @ r3 )�SuperWeirdWordPluginr Nc C s: d| _ d| _d| _d| _d| _d| _d| _d| _d| _d S )Nr F� ) � _word_count�_bad_word_count�_foreign_long_count�_is_current_word_bad�_foreign_long_watchr7 �_bad_character_count�_buffer�_buffer_accent_countr' r" r" r# r9 � s
z SuperWeirdWordPlugin.__init__r c C rP rQ r" r r" r" r# r$ rR z SuperWeirdWordPlugin.eligiblec C s |� � rH| j|7 _t|�r| jd7 _| jdu rFt|�du s%t|�rFt|�du rFt|�du rFt|�du rFt |�du rFt
|�du rFd| _d S | jsMd S |� � sYt |�sYt
|�r�| jr�| jd7 _t| j�}| j|7 _|dkr�| j| dkr}d| _t| jd �r�| jd �� r�| jd7 _d| _|dkr�| jr�dd � t| jtd
|��D �}d}|r�t|�| d kr�d}|s�| jd7 _d| _| jr�| jd7 _| jt| j�7 _d| _d| _d | _d
| _d S |d
v�r|�� du �rt|��rd| _| j|7 _d S d S d S d S )Nr FT� g�(\���?���� c S s g | ]
\}}|� � r|�qS r" )rZ )�.0�c�ir" r" r# �
<listcomp>/ s ��z-SuperWeirdWordPlugin.feed.<locals>.<listcomp>r rD rc > �_�<�>�~�|�=�-)rK rj r
rk rh r r
r r r r r^ r r rd �lenr7 rg rZ rf �zip�rangere ri r@ r )r! r Z
buffer_lengthZcamel_case_dstZprobable_camel_casedr" r" r# r& sr
����
�
��zSuperWeirdWordPlugin.feedc C s4 d| _ d| _d| _d| _d| _d| _d| _d| _d S )Nrc Fr )rj rg rh re rd r7 ri rf r' r" r" r# r( M s
zSuperWeirdWordPlugin.resetc C s$ | j dkr | jdkr dS | j| j S )N�
r rC )rd rf ri r7 r' r" r" r# r) W s zSuperWeirdWordPlugin.ratior* rE r" r" r" r# rb � s
A
rb c @ sZ e Zd ZdZddd�Zdedefdd�Zdeddfd d
�Zdd d �Z e
de fd
d��Z dS )�CjkInvalidStopPluginu�
GB(Chinese) based encoding often render the stop incorrectly when the content does not fit and
can be easily detected. Searching for the overuse of '丅' and '丄'.
r Nc C rG rA ��_wrong_stop_count�_cjk_character_countr' r" r" r# r9 e rJ z CjkInvalidStopPlugin.__init__r c C rP rQ r" r r" r" r# r$ i rR z CjkInvalidStopPlugin.eligiblec C s8 |dv r
| j d7 _ d S t|�r| jd7 _d S d S )N> u 丄u 丅r )r� r
r� r r" r" r# r& l s �zCjkInvalidStopPlugin.feedc C rG rA r r' r" r" r# r( s rJ zCjkInvalidStopPlugin.resetc C s | j dk rdS | j| j S )N� rC )r� r� r' r" r" r# r) w s
zCjkInvalidStopPlugin.ratior* )
r+ r, r- r. r9 r/ r0 r$ r& r( r1 r2 r) r" r" r" r# r~ _ s
r~ c @ r3 )�ArchaicUpperLowerPluginr Nc C s. d| _ d| _d| _d| _d| _d | _d| _d S )NFr T)�_buf� _character_count_since_last_sep� _successive_upper_lower_count�#_successive_upper_lower_count_finalr7 �_last_alpha_seen�_current_ascii_onlyr' r" r" r# r9 s
z ArchaicUpperLowerPlugin.__init__r c C rP rQ r" r r" r" r# r$ � rR z ArchaicUpperLowerPlugin.eligiblec C s$ |� � ot|�}|du }|rC| jdkrC| jdkr+|�� du r+| jdu r+| j| j7 _d| _d| _d | _d| _| j d7 _ d| _d S | jdu rQt
|�du rQd| _| jd ur|� � r_| j� � sh|� � r|| j� � r|| jdu rx| jd7 _d| _nd| _nd| _| j d7 _ | jd7 _|| _d S )NFr �@ r Tr? )
rK r r� r@ r� r� r� r� r� r7 r rZ �islower)r! r Z is_concernedZ chunk_sepr" r" r# r& � s@
�
��
z ArchaicUpperLowerPlugin.feedc C s. d| _ d| _d| _d| _d | _d| _d| _d S )Nr FT)r7 r� r� r� r� r� r� r' r" r" r# r( � s
z ArchaicUpperLowerPlugin.resetc C s | j dkrdS | j| j S )Nr rC )r7 r� r' r" r" r# r) � s
z ArchaicUpperLowerPlugin.ratior* rE r" r" r" r# r� ~ s
* r� � )�maxsizer` ra r c C sb | du s|du r
dS | |krdS d| v rd|v rdS d| v s"d|v r$dS d| v s,d|v r6d| v s4d|v r6dS | � d�|� d�}}|D ]}|tv rJqC||v rQ dS qC| dv |dv }}|s_|rid | v sgd |v ridS |ro|rodS d
| v swd
|v r�d | v sd |v r�dS | d ks�|d kr�dS d | v s�d |v s�| d v r�|d v r�d
| v s�d
|v r�dS d| v s�d|v r�dS dS )za
Determine if two Unicode range seen next to each other can be considered as suspicious.
NTFZLatinZ EmoticonsZ Combining� )�Hiragana�KatakanaZCJKZHangulz Basic Latin)r� r� Z PunctuationZForms)�splitr )r` ra Zkeywords_range_aZkeywords_range_b�elZrange_a_jp_charsZrange_b_jp_charsr" r" r# r_ � sZ �����r_ i 皙�����?F�decoded_sequence�maximum_threshold�debugc C sR dd� t �� D �}t| �d }d}|dk rd}n |dkr d}nd }t| d
t|��D ]2\}}|D ] } | �|�r<| �|� q0|d krG|| d ksM||d kr\td d
� |D ��}||kr\ nq*|r�td�}
|
� t
d|� d|� d|� �� t| �dkr�|
� t
d| dd� � �� |
� t
d| dd� � �� |D ]} |
� t
| j � d| j � �� q�t
|d�S )zw
Compute a mess ratio given a decoded bytes sequence. The maximum threshold does stop the computation earlier.
c S s g | ]}|� �qS r" r" )ro Zmd_classr" r" r# rr s �z mess_ratio.<locals>.<listcomp>r rC i � r� r� � �
r c s s � | ]}|j V qd S r; )r) )ro �dtr" r" r# � <genexpr>0 s � z mess_ratio.<locals>.<genexpr>Zcharset_normalizerzIMess-detector extended-analysis start. intermediary_mean_mess_ratio_calc=z mean_mess_ratio=z maximum_threshold=r� zStarting with: Nz
Ending with: i�z: � )r �__subclasses__rz r{ r| r$ r& �sumr �logr � __class__r) �round) r� r� r� Z detectors�lengthZmean_mess_ratioZ!intermediary_mean_mess_ratio_calcr �indexZdetector�loggerr� r" r" r# �
mess_ratio sN �
� �����
r� N)r� F)*� functoolsr �loggingr �typingr r Zconstantr r r �utilsr
r r r
r r r r r r r r r r r r r r4 rF rN rU r[ rb r~ r� r/ r0 r_ r2 r� r" r" r" r# �<module> sB H"/%4e L�� �F�����