o
    DDiq                     @   s(  U d dl mZ d dlZd dlZd dlZd dlmZ d dlm	Z	 d dl
mZ d dlmZmZmZmZmZmZmZ d dlmZ d dlmZmZ d	Zed
ZejZejZejZejZej Z ej!Z!ej"Z"ej#Z#ej$Z$ej%Z%ej&Z&ej'Z'ej(Z(ej)Z)ej*Z*eddG dd dZ+i Z,eee+f e-d< de.de/de.fddZ0de.de.fddZ1		dGdede2de2dee. fddZ3de.defddZ4dede+fd d!Z5e4d"Z6d#Z7e4d$e7 d% Z8e4d$e7 d& Z9e4d'Z:e4d(Z;dede+fd)d*Z<G d+d, d,e	d,g d-Z=G d.d/ d/e=Z>G d0d1 d1Z?d2d3 Z@d4d5 ZA	6dHd7e.ded8eeBeBf dee>ddf fd9d:ZC	6dHd;ee. ded8eeBeBf dee>ddf fd<d=ZD	6dHd;ee. ded>e+d8eeBeBf dee>ddf f
d?d@ZE	6dHd;ee. ded>e+d8eeBeBf dee>ddf f
dAdBZFdCe.d8eeBeBf dDe.dee>ddf fdEdFZGdS )I    )absolute_importN)BOM_UTF8)
namedtuple)	dataclass)Dict	GeneratorIterableOptionalPatternSetTuple)PythonTokenTypes)PythonVersionInfosplit_linesu   􏿿zutf-8T)frozenc                   @   sb   e Zd ZU eed< ee ed< ee ed< eeef ed< eed< eeef ed< ee ed< dS )	TokenCollectionpseudo_tokensingle_quotedtriple_quotedendpats
whitespacefstring_pattern_mapalways_break_tokensN)__name__
__module____qualname__r
   __annotations__r   strr    r   r   ]/var/www/Datamplify/venv/lib/python3.10/site-packages/libcst/_parser/parso/python/tokenize.pyr   ?   s   
 r   _token_collection_cachechoiceskwargsreturnc                  O   s6   | dd}|r
J d}|s|d7 }|d|  d S )NcaptureF(z?:|))popjoin)r!   r"   r$   startr   r   r   groupM   s   r+   c                  G   s   t |  d S )N?)r+   )r!   r   r   r   maybeW      r-   Fversion_infoinclude_fstringonly_fstringc           	      C   s   dd }ddg}| dkr| d | dk s| dkr| d d	h}| d
kr8|r8ddg}|r3|}t }n
||7 }n|r=t S |D ]}t|D ]	}||| qFq?| dkrc||d ||d |S )Nc                 s   s,    t jdd | D  D ]}d|V  qd S )Nc                 S   s   g | ]}||  fqS r   )upper).0cr   r   r   
<listcomp>b   s    zI_all_string_prefixes.<locals>.different_case_versions.<locals>.<listcomp> )
_itertoolsproductr)   )prefixsr   r   r   different_case_versionsa   s   z5_all_string_prefixes.<locals>.different_case_versionsbr   r   br)r?   r?   ur6   r?      ffr)      ur)appendsetr7   permutationsupdate)	r/   r0   r1   r;   valid_string_prefixesresultrD   r9   tr   r   r   _all_string_prefixes\   s.   


rP   exprc                 C   s   t | t jS N)recompileUNICODE)rQ   r   r   r   _compile      rV   c                 C   s2   zt |  W S  ty   t|  t | < }| Y S w rR   )r    KeyError_create_token_collection)r/   rN   r   r   r   _get_token_collection   s   
rZ   z(?:[^{}]+|\{\{|\}\})+z"[A-Za-z0-9\-]+(?: [A-Za-z0-9\-]+)*z(?:\{\{|\}\}|\\N\{z*\}|\\(?:\r\n?|\n)|\\[^\r\nN]|[^{}\r\n\\])+z\}|\\[^N]|[^{}\\])+z(?:\\(?:\r\n?|\n)|[^{}\r\n])+z[^{}]+c           +   	   C   s:  d}d}| j dkrd}ntjd dkrd}ndt d }| d	krKd
}d}d}d}t||||}d}	tddt|	 }
d|	 }t|
|}td|d }n:d}d}| dkrVd}nd}d}t||||}| j dk rj|d7 }d}	tddt|	 }
d|	 }t|
|}td |d }t|||}t| }t| }tt| d!d" }t| d!d!d#}t| }d$}d%}d&}d'}t|d( |d) }td*d+d,d-d.d/d0}d1}g d2}| dkr|dd3 | d4kr|dd5 n|dd6 t| }t|||}t|d7 td8d9 |d: td;d9 }||g}d<} |r||t|    td=g|R  }!t|d!d>t|!||||d!d> }"i }#|D ]#}$t	||#|$d8 < t	||#|$d; < t	||#|$d( < t	||#|$d) < q-t
 }%t
 }&i }'|D ]}(d?D ]
})|%|(|)  q_d@D ]
})|&|(|)  qlq[|D ]}(| D ]	})|)|'|(|) < qq{t	|"}*t|*|%|&|#t	||'h dAS )BNz[ \f\t]*z	#[^\r\n]*rF   z([A-Za-z_0-9]+)r   z(\w+)u   ([A-Za-z_0-9-z]+)rB   z0[xX](?:_?[0-9a-fA-F])+z0[bB](?:_?[01])+z0[oO](?:_?[0-7])+z(?:0(?:_?0)*|[1-9](?:_?[0-9])*)z[eE][-+]?[0-9](?:_?[0-9])*z)[0-9](?:_?[0-9])*\.(?:[0-9](?:_?[0-9])*)?z\.[0-9](?:_?[0-9])*z[0-9](?:_?[0-9])*z[0-9](?:_?[0-9])*[jJ]z[jJ]z0[xX][0-9a-fA-F]+z
0[bB][01]+r>   z0[oO][0-7]+z0[oO]?[0-7]+z(?:0+|[1-9][0-9]*)r?   z[lL]?z[eE][-+]?[0-9]+z[0-9]+\.[0-9]*z\.[0-9]+z[0-9]+z
[0-9]+[jJ]T)r0   )r0   r1   z(?:\\.|[^'\\])*'z(?:\\.|[^"\\])*"z(?:\\.|'(?!'')|[^'\\])*'''z(?:\\.|"(?!"")|[^"\\])*"""'''"""z\*\*=?z>>=?z<<=?z//=?z->z[+\-*/%&@`|^!=<>]=?~z[][(){}])z\r\n?z\nz[;.,@]z\.\.\.)r?      z:=?:z'[^\r\n'\\]*(?:\\.[^\r\n'\\]*)*'z\\(?:\r\n?|\n)z"[^\r\n"\\]*(?:\\.[^\r\n"\\]*)*")ra   r`   r\   r[   z\\(?:\r\n?|\n)|\Z)r$   )ra   r`   )r\   r[   >
   deftrywithclasswhileexceptimportr#   finally;)majorsysr/   MAX_UNICODEr+   r-   rP   insertrI   rV   rJ   addr   )+r/   
WhitespaceCommentName	Hexnumber	Binnumber	Octnumber	Decnumber	IntnumberExponent
PointfloatExpfloatFloatnumber
ImagnumberNumberpossible_prefixesStringPrefixStringPrefixWithFfstring_prefixesFStringStartSingleDoubleSingle3Double3TripleOperatorBracketspecial_argsSpecialFunnyContStrpseudo_extra_pool
all_quotesPseudoExtrasPseudoTokenr   _prefixr   r   r   rO   quotepseudo_token_compiledr   r   r   rY      s   



rY   c                   @   s   e Zd Zedd ZdS )Tokenc                 C   sN   t | j}t|dkr| jd t| d dfS | jd | jd t| j fS N   r   )r   stringlen	start_pos)selflinesr   r   r   end_posB  s   
zToken.end_posN)r   r   r   propertyr   r   r   r   r   r   A  s    r   )typer   r   r9   c                   @   s   e Zd Zdd ZdS )PythonTokenc                 C   s   d| j | jjd S )Nz6TokenInfo(type=%s, string=%r, start_pos=%r, prefix=%r))r   )_replacer   namer   r   r   r   __repr__L  s   zPythonToken.__repr__N)r   r   r   r   r   r   r   r   r   K  s    r   c                   @   s<   e Zd Zdd Zdd Zdd Zdd Zd	d
 Zdd ZdS )FStringNodec                 C   s(   || _ || _d| _d| _d | _d| _d S )Nr   r6   )r   rawparentheses_countprevious_lineslast_string_start_posformat_spec_count)r   r   r   r   r   r   __init__S  s   
zFStringNode.__init__c                 C   s   |  j d7  _ d S )Nr   )r   r   	characterr   r   r   open_parentheses]  s   zFStringNode.open_parenthesesc                 C   s&   |  j d8  _ | j dkrd| _d S d S r   r   r   r   r   r   r   close_parentheses`  s   

zFStringNode.close_parenthesesc                 C   s   t | jdkS )Nr?   )r   r   r   r   r   r   allow_multilinef  rW   zFStringNode.allow_multilinec                 C   s   | j | jkS rR   r   r   r   r   r   
is_in_expri  r.   zFStringNode.is_in_exprc                 C   s   |    o| jS rR   )r   r   r   r   r   r   is_in_format_specl  s   zFStringNode.is_in_format_specN)	r   r   r   r   r   r   r   r   r   r   r   r   r   r   R  s    
r   c                 C   sf   t | D ])\}}||jr-tt|j||d}d}|jrJ | |d = |dt|jf  S qd |dfS )Nr9   r6   r   )	enumerate
startswithr   r   FSTRING_ENDr   r   )fstring_stackr   r   additional_prefixfstring_stack_indexnodetokenr   r   r   _close_fstring_if_necessaryp  s   



r   c                 C   s  |d }|  }| r|rt}nt}n|jrt}n|rt}nt}|||}|d u r0|j	|fS |j	s8||f|_
|d}	|D ]}
| |
j |	}|d urZ|dd t|
j  }	q?|}|t|	7 }|	dsm|	drz| j	|	7  _	d}	|	|fS |j	|	 }	|	|fS )Nr   
r6   )r   r   fstring_format_spec_multi_linefstring_format_spec_single_liner   fstring_raw_stringfstring_string_multi_linefstring_string_single_linematchr   r   r+   r   r   endswith)r   r   linelnumpostosr   regexr   r   fstring_stack_node	end_matchnew_posr   r   r   _find_fstring_string}  s<   



r   r   r   coder   c                 C   s   t | dd}t|||dS )z0Generate tokens from a the source code (string).T)keependsr   )r   tokenize_lines)r   r/   r   r   r   r   r   tokenize  s   r   r   c                 C   s6   t |}|tddkrt| |||dS t| |||dS )Nr?   rG   r   )rZ   r   _tokenize_lines_py37_or_above_tokenize_lines_py36_or_below)r   r/   r   token_collectionr   r   r   r     s   r   r   c           #   	   #   sH	   d}dgd}d}d}d}d}	d}
d}d}d}d}|d d g }dd dd fdd	}| D ]'}d7 d}t |}|rl|trXt}|dd }t |}d
|d  | }|d }||d 7 }d}|r|du rvtd||}|r|d}|du rtddurtdtt||d|  ||
V  d}d}n	|| }|| }q8||k r`|r|d }| st	|j
|||\}}|rdurtdtt||jddV  d|_q||krny||d }t|||f|\}}}||7 }|durdurtd|V  q|j||}|sL|j||}|dkr.|| E dH  | }d}	tt|| |f||d V  d}|d7 }q||d }
d}|d\}}|f|d}|dkrs|
soJ |
}n|d }|	r|dvrd}	|dkr|sd}|}|| dkr|d7 }|d8 }|| dks|d krdurV  dttddV  | ||E dH  ||v s|dkr|dkr|dkrdurV  dtt||
V  nl|ddur||jv rGg |dd< d}td|d| rG	  } | |kr? r,r,| kr,d dddur6V  dttddV  n|  nqt|rd}!|dv rp rp|dkrett||
V  ntt||
V  d}!nI|dkrstt||
d}!n7|dkrdurd tu rd dkrd d ttd d d V  dnrV  d|!rtt||
V  nt ||
E dH  n|dv rt!dd |D rg |dd< |	s|dkr|s rdrV  dtt"||
V  n|
| }d}	nM|dkr"|#drJ |
| }n:||j$v rb|j
| }|||}|rU|d}||| }durKV  dtt||
V  n|f}||d }|}n||j%v s||dd |j%v s||dd |j%v r|d dv r|f}|j
&|p|j
&|d p|j
&|d }||d }|}ndurV  dtt||
V  n||j'v r|t(|j'| d|v pd|v  dur݈V  dtt)||
V  nv|dkr||d d v r||
||d  7 }na|d!v r|r|d *| n;|d7 }n6|d"v r,|r$|d +| n&|r+|d8 }n|d#krJ|rJ|d j,|d j- dkrJ|d  j-d7  _-durTV  dtt.||
V  ||k sq8|rztt|||
V  |#dsx|#d$rzd}	durV  d|f}"dd D ]} ttd|"dV  qtt/d|"|V  dS )%)  
    A heavily modified Python standard library tokenizer.

    Additionally to the default information, yields also the prefix of each
    token. This idea comes from lib2to3. The prefix contains all information
    that is irrelevant for the parser like newlines in parentheses or comments.
    r   
0123456789r6   NTr   Fc                 3   s    | d k rF| d krt tddfdV  d S d ur"V  d  r2r2d kr2d ddt tddV    | d k sd S d S )Nr   r6   r   Fr   ERROR_DEDENTDEDENTr(   r*   	async_defasync_def_indentasync_def_newlineindentsr   sposstashedr   r   dedent_if_necessary  s   z:_tokenize_lines_py36_or_below.<locals>.dedent_if_necessary^Logic error!r   r   rF   
\#....r?   	[ \f\t]*$)asyncawaitr   rb   
c                 s       | ]}|   V  qd S rR   r   r3   rD   r   r   r   	<genexpr>      z0_tokenize_lines_py36_or_below.<locals>.<genexpr>#r   r=   R\z\
z\
z\([{)]}r_   r   0r   r   BOM_UTF8_STRING	Exceptionr   endr   STRINGr   r   r   FSTRING_STRINGr   r   r   r   r   
ERRORTOKENr+   spanINDENTrI   NUMBERr   rS   r(   r   r   isidentifierASYNCAWAITNAME_split_illegal_unicode_nameanyNEWLINEr   r   r   getr   r   FSTRING_STARTr   r   r   r   OP	ENDMARKER)#r   r/   r   r   paren_levelmaxnumcharscontstrcontlinenew_liner9   endprogcontstr_startr   firstr   r   r   r   endmatchr   r   restfstring_end_tokenquote_lengthpseudomatchr   r*   r   initialiindent_startindentshould_yield_identifierr   r   r   r   r     s.  

































    
r   c           #   	   #   s    fdd}d}dg d}d}d}d}	d}
d}d}d}d}d}|d d g }| D ]b}d7 d}t |}|r`|trLt}|dd }t |}d	|d  | }|d }||d 7 }d
}|r|du rjtd||}|r|d}|du r~tdtt||d|  ||V  d}d}	n	|| }|	| }	q,||k r|r|d }| st	|j
|||\}}|rtt||jddV  d|_q||krʐn||d }t|||f|\}}}||7 }|dur|V  q|j||}|s)|j||}|dkr|| D ]}|V  q| }d
}
tt|| |f||d V  d}|d7 }q||d }d}|d\}}|f|d}|dkrP|sLJ |}n?|d }|
r|dvrd
}
|dkr|sd}|} || dkr|d7 }| d8 } || dksq|  d krttddV   |  || D ]}|V  q||v s|dkr|dkr|dkrtt||V  n|ddur6||jv rg |dd< d}td|d| r	   }!|!|krttddV  n |! nqt|r'|dkrtt||V  n}|dkrtt||V  nntt||V  ndt ||D ]}|V  q-nU|dv rgt!dd |D rKg |dd< |
s_|dkr_|s_tt"||V  n|| }d}
n$|dkrz|#drtJ || }n||j$v r|j
| }|||}|r|d}||| }tt||V  n|f}||d }|}	n||j%v s|dd |j%v s|dd |j%v r|d dv r|f}|j
&|p|j
&|d p|j
&|d }||d }|}	ntt||V  n||j'v r|t(|j'| d|v pd|v  tt)||V  nl|dkr8||d dv r8||||d  7 }nW|d v rM|rH|d *| n;|d7 }n6|d!v re|r]|d +| n&|rd|d8 }n|d"kr|r|d j,|d j- dkr|d  j-d7  _-tt.||V  ||k sq,|rtt|||V  |#ds|#d#rd}
|f}" dd D ]}!ttd|"dV  qtt/d|"|V  dS )$r   c                 3   s^    |  d k r-|  d krt tddfdV  d S t tddV     |  d k sd S d S )Nr   r   r6   r   r   r   r   r   r   r   r   r   K  s   z:_tokenize_lines_py37_or_above.<locals>.dedent_if_necessaryr   r   r6   NTr   r   Fr   r   r   rF   r   r   r   r   r?   r   r   r   r   c                 s   r   rR   r   r   r   r   r   r     r   z0_tokenize_lines_py37_or_above.<locals>.<genexpr>r   r   r=   r   r   r   r   r   r_   r   r   )#r   r/   r   r   r   r  r  r  r  r  r  r9   r  r  r   r  r   r   r   r  r   r   r  r  r  r   r   rO   r*   r   r!  r"  r#  r$  r   r   r&  r   r   =  s  

























  Pr   r   r9   c                 #   s     fdd}d d|t | D ]G\}}r9t|r4| V  | dd|d |d | fq |7  q | }t|rE| q rW| V  d|d |d | f| dq rd| V  d S d S )Nc                      s   t rtnt S rR   )r   r  r  r   found
is_illegalr   r9   r   r   create_tokene  s   z1_split_illegal_unicode_name.<locals>.create_tokenr6   Fr   r   T)r   r   r  )r   r   r9   r*  r"  char	new_foundr   r'  r   r  b  s4   


r  )FF)r   )H
__future__r   	itertoolsr7   rS   rl   codecsr   collectionsr   dataclassesr   typingr   r   r   r	   r
   r   r   !libcst._parser.parso.python.tokenr   libcst._parser.parso.utilsr   r   rm   decoder   r  r  r  r  r  r  r   r	  r
  r  r  r   r  r  r   r   r    r   r   objectr+   r-   boolrP   rV   rZ   r   unicode_character_namer   r   r   r   rY   r   r   r   r   r   intr   r   r   r   r  r   r   r   r   <module>   s   $



*


 
+





   

  '
