o
    DDi6                     @   s   d dl mZ d dlmZ d dlmZmZ d dlmZm	Z	 d dl
mZmZ ejZejZejZejZejZejZejZejZejZejZejZejZejZdddZG d	d
 d
eZdS )    )dedent)PythonTokenTypes)PythonTokentokenize)parse_version_stringsplit_lines)data_providerUnitTestNc                 C   s   t |}tt| |S N)r   listr   )stringversionversion_info r   a/var/www/Datamplify/venv/lib/python3.10/site-packages/libcst/_parser/parso/tests/test_tokenize.py_get_token_list)   s   r   c                   @   sH  e Zd Zdd Zdd Zdd Zdd Zd	d
 Zdd Zdd Z	dd Z
dd Zdd Zedeeegfdeeeeeegfdeeeeeeeeeg	fdeeeeegfdeegfdegfdegfdeeegfdeeegff	dd Zd d! Zd"d# Zd$d% Zd&d' Zd(d) Zd*d+ Zd,d- Zed.egd/fd0eegd/fd1eee e egd/fd2eeee e gd/fd3eeegd/fd3eeegd/fd4eee ee ee e ee egd/fd5eeegd/fd6eee ee eegd/fd7eeegd/fd8eee ee eegd/fd9ee ee egd/fd:ee ee ee egd/fd;eeeeegd/fd<ee eee ee ee ee ee ee gd/ffd=d> Z!d?S )@ParsoTokenizerTestc                 C   s8   d}t |}|d \}}}}|dksJ |dksJ d S )N"""simple one line docstring"""r    )r   )selfsimple_docstring
token_list_valueprefixr   r   r   test_simple_no_whitespace0   s
   z,ParsoTokenizerTest.test_simple_no_whitespacec                 C   s   d}t |}|d d tksJ |d \}}}}|dksJ |dks$J |tks*J |d \}}}}|dks8J |tks>J d S )Nz$  """simple one line docstring""" 
r      z  r       )r   INDENTSTRINGNEWLINE)r   r   r   typr   	start_posr   r   r   r   test_simple_with_whitespace8   s   z.ParsoTokenizerTest.test_simple_with_whitespacec                 C   s   t d}t|}|D ]B\}}}}|dkr|dksJ |dkr$|dks$J |dkr.|dks.J |dkr8|dks8J |dkrB|d	ksBJ |d
krL|dksLJ q
d S )Nz{
        def test_whitespace(*args, **kwargs):
            x = 1
            if x > 0:
                print(True)
        test_whitespacer   (r   *z**printz        ifz    )r   r   )r   fundefr   r   r   r   r   r   r   test_function_whitespaceE   s&   z+ParsoTokenizerTest.test_function_whitespacec                 C   4   d}t |}|ttdddttdddgksJ d S )Nz""""
r   r   r   )r   r   r   r   
ERRORTOKEN	ENDMARKERr   r*   r   r   r   r   test_tokenize_multiline_I^      z,ParsoTokenizerTest.test_tokenize_multiline_Ic                 C   r,   )Nz""""r-   r   )r      r.   r1   r   r   r   test_tokenize_multiline_IIh   r3   z-ParsoTokenizerTest.test_tokenize_multiline_IIc                 C   r,   )Nz""""

r-   r   )   r   r.   r1   r   r   r   test_tokenize_multiline_IIIr   r3   z.ParsoTokenizerTest.test_tokenize_multiline_IIIc                 C   s,   t d}t|}|d }|d tksJ d S )Nu2   
        def 我あφ():
            pass
        r   r   )r   r   NAME)r   r*   r   unicode_tokenr   r   r    test_identifier_contains_unicode|   s   z3ParsoTokenizerTest.test_identifier_contains_unicodec                 C   sz   ddd}|d |ddd |ddd |d	dd |d
 |d |ddd |ddd |ddd |ddd dS )at  
        Decided to parse `u''` literals regardless of Python version. This makes
        probably sense:

        - Python 3+ doesn't support it, but it doesn't hurt
          not be. While this is incorrect, it's just incorrect for one "old" and in
          the future not very important version.
        - All the other Python versions work very well with it.
        Tc                 S   sT   t | }|d \}}}}|r"|tkr|tksJ || ks J d S d S |tks(J d S )Nr   )r   FSTRING_STARTr    r8   )literal
is_literalr   r"   result_literalr   r   r   r   check   s   z2ParsoTokenizerTest.test_ur_literals.<locals>.checkzu""zur""F)r=   zUr""zUR""zbR""zRb""zfr""zrF""f""zF""N)Tr   r   r?   r   r   r   test_ur_literals   s   

z#ParsoTokenizerTest.test_ur_literalsc                 C   s   t d\}}}|jtksJ |jdksJ |jtksJ |jtks#J |jdks*J t d\}}}|jtks8J |jdks?J |jdksFJ |jtksMJ |jdksTJ d S )Nz"
"r   z( """r   """)r   typer/   r   r!   r0   r   )r   error_tokennewline	endmarkerbracketr   r   r   test_error_literal   s   z%ParsoTokenizerTest.test_error_literalc                 C   s<   dd }|d |d |d |d |d |d d S )	Nc                 S   s6   t | }t| }|d jt|t|d fksJ d S )N)r   r   end_poslen)codetokenslinesr   r   r   r?      s   &z8ParsoTokenizerTest.test_endmarker_end_pos.<locals>.checkz#cz#c
za
aza\\nza\r   rA   r   r   r   test_endmarker_end_pos   s   z)ParsoTokenizerTest.test_endmarker_end_posz fooz
  foo
 barz  foo
 bar 
 bazz foo
bar1foo1u   மெல்லினம்   ²u   ä²öu   ää²¹ööc                 C   s(   dd t |D }||tg ksJ d S )Nc                 S      g | ]}|j qS r   rE   .0tr   r   r   
<listcomp>       z7ParsoTokenizerTest.test_token_types.<locals>.<listcomp>r   r0   )r   rN   typesactual_typesr   r   r   test_token_types   s   z#ParsoTokenizerTest.test_token_typesc                 C   sf   t d\}}}|jtksJ |jdksJ |jdksJ |jtks#J |jdks*J |jdks1J d S )Nz "
r   rC   r   )r   rE   r/   r   r   r!   )r   t1rG   rH   r   r   r   test_error_string   s   z$ParsoTokenizerTest.test_error_stringc                 C   sF   t d}t|}ttttttttttttg}dd |D |ks!J d S )NzY                            str(
            from x import a
            def
            c                 S   rU   r   rV   rW   r   r   r   rZ     r[   zAParsoTokenizerTest.test_indent_error_recovery.<locals>.<listcomp>)r   r   r   r8   OPr!   DEDENTr0   r   rN   lstexpectedr   r   r   test_indent_error_recovery   s$   z-ParsoTokenizerTest.test_indent_error_recoveryc                 C   sF   t d}t|}ttttttttttttg}dd |D |ks!J d S )NzG            class C:
                pass
            $foo
            c                 S   rU   r   rV   rW   r   r   r   rZ   &  r[   zDParsoTokenizerTest.test_error_token_after_dedent.<locals>.<listcomp>)	r   r   r8   rb   r!   r   rc   r/   r0   rd   r   r   r   test_error_token_after_dedent  s$   z0ParsoTokenizerTest.test_error_token_after_dedentc                 C   s6   t d}t|}dd |D ttttttgksJ dS )z~
        There used to be an issue that the parentheses counting would go below
        zero. This should not happen.
        z8            }
            {
              }
            c                 S   rU   r   rV   rW   r   r   r   rZ   5  r[   zCParsoTokenizerTest.test_brackets_no_indentation.<locals>.<listcomp>N)r   r   rb   r!   r0   )r   rN   re   r   r   r   test_brackets_no_indentation(  s
   &z/ParsoTokenizerTest.test_brackets_no_indentationc                 C   s>   t td\}}|jdksJ |jdksJ |jdksJ d S )Nz            """rD   r   )r   r   r   r   )r   rF   rH   r   r   r   test_form_feed7  s   z!ParsoTokenizerTest.test_form_feedc                 C   s,   t d}dd |D tttttgksJ d S )Nz	 =\classc                 S   rU   r   rV   rW   r   r   r   rZ   D  r[   z;ParsoTokenizerTest.test_carriage_return.<locals>.<listcomp>)r   r   rb   rc   r8   r0   )r   re   r   r   r   test_carriage_returnB  s   $z'ParsoTokenizerTest.test_carriage_returnc                 C   s    d}t |\}|j|ksJ d S )Nz\
# 1 
)r   r   )r   rN   rH   r   r   r   test_backslashF  s   
z!ParsoTokenizerTest.test_backslashzf"z3.7r@   zf" {}"zf" "{}zf"\""zf"Some {x:.2f}{y}"zf"""abc
def"""zf"""abc{
123}def"""zf"abc\
def"zf"\
{123}\
"z
f"{\
123}"zf"{123:.2\
f}"z
f"abc
def"zprint(f"Some {x:.2f}a{y}")c                 C   s*   dd t ||D }|tg |ksJ d S )Nc                 S   rU   r   rV   rW   r   r   r   rZ     r[   z3ParsoTokenizerTest.test_fstring.<locals>.<listcomp>r\   )r   rN   r]   
py_versionr^   r   r   r   test_fstringK  s   `zParsoTokenizerTest.test_fstringN)"__name__
__module____qualname__r   r$   r+   r2   r5   r7   r:   rB   rJ   rR   r   r   r8   rc   r!   ERROR_DEDENTNUMBERr/   r_   ra   rg   rh   ri   rk   rl   rm   r;   FSTRING_ENDFSTRING_STRINGrb   ro   r   r   r   r   r   /   s    


#

	
		_r   r
   )textwrapr   !libcst._parser.parso.python.tokenr   $libcst._parser.parso.python.tokenizer   r   libcst._parser.parso.utilsr   r   libcst.testing.utilsr   r	   r8   r!   r    rt   r   rc   r/   rb   r0   rs   r;   rv   ru   r   r   r   r   r   r   <module>   s(   
