[EDIT] FILE: tokenize.cpython-36.opt-2.pyc
3 \NX � = @ s� d Z dZddlZddlZddlmZmZ ddlT ddlm Z dd � e e �D �d d d g Z [ ye W n e k rz eZ Y nX d d� Zdd� Zdd� ZdZdZeede � ee� ZdZdZdZdZedd�Zeeeee�Zd Zed d �ee� Z d e Z ee e �Z ed e d! �Z ee e e�Z d"Z!d#Z"d$Z#d%Z$d&Z%ee%d' e%d( �Z&ee%d) e%d* �Z'ed+d,d-d.d/d0d1d2d3� Z(d4Z)ed5d6�Z*ee(e)e*�Z+ee e+e'e�Z,ee, Z-ee%d7 ed8d� e%d9 ed:d� �Z.edee&�Z/eee/e e+e.e� Z0e1e2ej3e-e0e#e$f��\Z4Z5Z6Z7ej3e!�ej3e"�e6e7e6e7e6e7e6e7e6e7e6e7e6e7e6e7e6e7e6e7e6e7e6e7e6e7e6e7e6e7e6e7e6e7e6e7e6e7e6e7e6e7ddddddddd;�4Z8i Z9xd�D ]Z:e:e9e:< �q�W i Z;xd�D ]Z:e:e;e:< �q�W d�Z<G d�d�� d�e=�Z>G d�d�� d�e=�Z?d�d�� Z@e@fd�d �ZAd�d�� ZBG d�d�� d��ZCej3d�ejD�ZEej3d�ejD�ZFd�d�� ZGd�d�� ZHd�d � ZId�d � ZJeKd�k�r�ddlLZLeMeLjN�dk�rxeAeOeLjNd �jP� n eAeLjQjP� dS )�zKa-Ping Yee <ping@lfw.org>z@GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, Skip Montanaro� N)�BOM_UTF8�lookup)�*� )�tokenc C s g | ]}|d dkr|�qS )r �_� )�.0�xr r �./usr/lib64/python3.6/lib2to3/pgen2/tokenize.py� <listcomp>% s r �tokenize�generate_tokens� untokenizec G s ddj | � d S )N�(�|�))�join)�choicesr r r �group0 s r c G s t | � d S )Nr )r )r r r r �any1 s r c G s t | � d S )N�?)r )r r r r �maybe2 s r z[ \f\t]*z #[^\r\n]*z\\\r?\nz [a-zA-Z_]\w*z0[bB]_?[01]+(?:_[01]+)*z(0[xX]_?[\da-fA-F]+(?:_[\da-fA-F]+)*[lL]?z 0[oO]?_?[0-7]+(?:_[0-7]+)*[lL]?z[1-9]\d*(?:_\d+)*[lL]?z0[lL]?z[eE][-+]?\d+(?:_\d+)*z \d+(?:_\d+)*\.(?:\d+(?:_\d+)*)?z\.\d+(?:_\d+)*z \d+(?:_\d+)*z\d+(?:_\d+)*[jJ]z[jJ]z[^'\\]*(?:\\.[^'\\]*)*'z[^"\\]*(?:\\.[^"\\]*)*"z%[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''z%[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""z#(?:[uUrRbBfF]|[rR][bB]|[bBuU][rR])?�'''�"""z '[^\n'\\]*(?:\\.[^\n'\\]*)*'z "[^\n"\\]*(?:\\.[^\n"\\]*)*"z\*\*=?z>>=?z<<=?z<>z!=z//=?z->z[+\-*/%&@|^=<>]=?�~z[][(){}]z\r?\nz[:;.,`@]z'[^\n'\\]*(?:\\.[^\n'\\]*)*�'z"[^\n"\\]*(?:\\.[^\n"\\]*)*�")4r r z'''z"""zr'''zr"""zu'''zu"""zb'''zb"""zf'''zf"""zur'''zur"""zbr'''zbr"""zrb'''zrb"""zR'''zR"""zU'''zU"""zB'''zB"""zF'''zF"""zuR'''zuR"""zUr'''zUr"""zUR'''zUR"""zbR'''zbR"""zBr'''zBr"""zBR'''zBR"""zrB'''zrB"""zRb'''zRb"""zRB'''zRB"""�r�R�u�U�f�F�b�B�r'''�r"""�R'''�R"""�u'''�u"""�U'''�U"""�b'''�b"""�B'''�B"""�f'''�f"""�F'''�F"""�ur'''�ur"""�Ur'''�Ur"""�uR'''�uR"""�UR'''�UR"""�br'''�br"""�Br'''�Br"""�bR'''�bR"""�BR'''�BR"""�rb'''�rb"""�Rb'''�Rb"""�rB'''�rB"""�RB'''�RB"""�r'�r"�R'�R"�u'�u"�U'�U"�b'�b"�B'�B"�f'�f"�F'�F"�ur'�ur"�Ur'�Ur"�uR'�uR"�UR'�UR"�br'�br"�Br'�Br"�bR'�bR"�BR'�BR"�rb'�rb"�Rb'�Rb"�rB'�rB"�RB'�RB"� c @ s e Zd ZdS )� TokenErrorN)�__name__� __module__� __qualname__r r r r rw � s rw c @ s e Zd ZdS )�StopTokenizingN)rx ry rz r r r r r{ � s r{ c C s4 |\}}|\}}t d||||t| t|�f � d S )Nz%d,%d-%d,%d: %s %s)�print�tok_name�repr) �typer Zxxx_todo_changemeZxxx_todo_changeme1�lineZsrowZscolZerowZecolr r r � printtoken� s r� c C s( yt | |� W n tk r" Y nX d S )N)� tokenize_loopr{ )�readline� tokeneaterr r r r � s c C s xt | �D ] }||� q W d S )N)r )r� r� Z token_infor r r r� � s r� c @ s, e Zd Zdd� Zdd� Zdd� Zdd� Zd S ) � Untokenizerc C s g | _ d| _d| _d S )Nr r )�tokens�prev_row�prev_col)�selfr r r �__init__� s zUntokenizer.__init__c C s* |\}}|| j }|r&| jjd| � d S )N� )r� r� �append)r� �start�row�col� col_offsetr r r �add_whitespace� s zUntokenizer.add_whitespacec C s� xv|D ]n}t |�dkr$| j||� P |\}}}}}| j|� | jj|� |\| _| _|ttfkr| jd7 _d| _qW dj | j�S )N� r r � ) �len�compatr� r� r� r� r� �NEWLINE�NLr )r� �iterable�t�tok_typer r� �endr� r r r r � s zUntokenizer.untokenizec C s� d}g }| j j}|\}}|ttfkr,|d7 }|ttfkr<d}x�|D ]�}|d d� \}}|ttttfkrn|d7 }|tkr�|j|� qBn>|t kr�|j � qBn*|ttfkr�d}n|r�|r�||d � d}||� qBW d S )NFr� Tr� r ���) r� r� �NAME�NUMBERr� r� �ASYNC�AWAIT�INDENT�DEDENT�pop) r� r r� � startline�indents� toks_append�toknum�tokval�tokr r r r� � s0 zUntokenizer.compatN)rx ry rz r� r� r r� r r r r r� � s r� z&^[ \t\f]*#.*?coding[:=][ \t]*([-\w.]+)s ^[ \t\f]*(?:[#\r\n]|$)c C sH | d d� j � jdd�}|dks*|jd�r.dS |d ks@|jd �rDdS | S )N� r �-zutf-8zutf-8-�latin-1� iso-8859-1� iso-latin-1�latin-1-� iso-8859-1-� iso-latin-1-)r� r� r� )r� r� r� )�lower�replace� startswith)�orig_enc�encr r r �_get_normal_name s r� c s� d� d }d}�fdd�}� fdd�}|� }|j t�rHd� |dd � }d }|sT|g fS ||�}|rj||gfS tj|�s~||gfS |� }|s�||gfS ||�}|r�|||gfS |||gfS ) NFzutf-8c s" y� � S t k r t� S X d S )N)� StopIteration�bytesr )r� r r � read_or_stop s z%detect_encoding.<locals>.read_or_stopc s� y| j d�}W n tk r" d S X tj|�}|s6d S t|jd��}y t|�}W n tk rp td| ��Y nX � r�|j dkr�td��|d7 }|S )N�asciir zunknown encoding: zutf-8zencoding problem: utf-8z-sig) �decode�UnicodeDecodeError� cookie_re�matchr� r r � LookupError� SyntaxError�name)r� � line_stringr� �encoding�codec)� bom_foundr r � find_cookie& s" z$detect_encoding.<locals>.find_cookieT� z utf-8-sig)r� r �blank_rer� )r� r� �defaultr� r� �first�secondr )r� r� r �detect_encoding s0 r� c C s t � }|j| �S )N)r� r )r� �utr r r r T s c ! c s� d } }}t jd d }}d \}}d }dg} d } d} d} d} �x�y | � }W n tk rd d}Y nX |d }dt|� }}|�rF|s�td|��|j|�}|r�|jd� }}t||d |� |||f|| fV d \}}d }nd|�r0|d!d � d k�r0|d"d � d k�r0t|| ||t|�f|fV d}d }qBn|| }|| }qB�nF|dk�rt| �rt|�s`P d}xf||k �r�|| d k�r�|d }n6|| d k�r�|t d t }n|| dk�r�d}nP |d }�qfW ||k�r�P | �r�| V d } || dk�r�|| dk�rh||d � j d�}|t|� }t |||f||t|� f|fV t ||d � ||f|t|�f|fV qBt t f|| dk ||d � ||f|t|�f|fV qB|| d# k�r�| j |� t|d |� |df||f|fV xt|| d$ k �rJ|| k�rtdd|||f��| d d%� } | �r.| | d& k�r.d} d} d} td||f||f|fV �q�W | �r�| �r�| | d' k�r�d} d} d} n|�s�td|df��d}�x�||k �r8tj||�}|�r |jd�\}}||f||f| }}}|||� || } } | |k�s| dk�r| dk�rt| |||fV �q4| dk�rft} |dk�r8t } n | �rBd} | �rR| V d } | | |||fV �q4| dk�r�| �r�| V d } t | |||fV �q4| tk�rt| }|j||�}|�r�|jd�}|||� } | �r�| V d } t| |||f|fV n||f}||d � }|}P �q4| tk�s@| d d� tk�s@| d d � tk�r�| d( dk�r�||f}t| �pxt| d �pxt| d }||d � d }}|}P n | �r�| V d } t| |||fV �q4| |k�r�| d)k�r�| �r�| dk�r�tnt| |||fV �q�t| |||f} | dk�r | �r | } �q�| dk�rx| �rx| d tk�rx| d dk�rxd} | d* } t| d | d | d | d fV d } | �r�| V d } | V nz| d k�r�| �r�| V d } t | |||f|fV d}nF| d k�r�|d }n| d k�r�|d }| �r�| V d } t| |||fV n(t|| ||f||d f|fV |d }�q�W qBW | �rN| V d } x.| dd � D ] } td|df|dfdfV �q\W t d|df|dfdfV d S )+Nr r � 0123456789r� Fr zEOF in multi-line stringr� z\ r� z\ r� � � z# �#z z3unindent does not match any outer indentation levelz <tokenize>zEOF in multi-line statement�.T� �async�await�def� �\z([{z)]})r� r )r� r ������r� r� r� r� r� r� )r� r� r� ) �stringZ ascii_lettersr� r� rw r� r� �STRING� ERRORTOKEN�tabsize�rstrip�COMMENTr� r� r� �IndentationErrorr� � pseudoprog�spanr� r� � triple_quoted�endprogs� single_quotedr� r� r� �OP� ENDMARKER)!r� �lnum�parenlev� continuedZ namechars�numchars�contstr�needcont�contliner� �stashed� async_def�async_def_indent� async_def_nlr� �pos�max�strstart�endprog�endmatchr� �column� comment_token�nl_pos� pseudomatchr� �spos�eposr �initial�newliner� �indentr r r r i sp * �__main__)*r r r& r' r( r) r* r+ r, r- r. r/ r0 r1 r2 r3 r4 r5 r6 r7 r8 r9 r: r; r< r= r> r? r@ rA rB rC rD rE rF rG rH rI rJ rK rL rM )*r r rN rO rP rQ rR rS rT rU rV rW rX rY rZ r[ r\ r] r^ r_ r` ra rb rc rd re rf rg rh ri rj rk rl rm rn ro rp rq rr rs rt ru )R� __author__� __credits__r� �re�codecsr r Zlib2to3.pgen2.tokenr� r �dir�__all__r� � NameError�strr r r � Whitespace�Comment�Ignore�Name� Binnumber� Hexnumber� Octnumber� Decnumber� Intnumber�Exponent� Pointfloat�Expfloat� Floatnumber� Imagnumber�Number�Single�Double�Single3�Double3Z _litprefix�Triple�String�Operator�Bracket�Special�Funny� PlainToken�Token�ContStr� PseudoExtras� PseudoToken�list�map�compileZ tokenprogr� Z single3progZ double3progr� r� r� r� r� � Exceptionrw r{ r� r r� r� �ASCIIr� r� r� r� r r rx �sysr� �argv�openr� �stdinr r r r �<module> s� 8 I c
SAVE
CANCEL