B
    18\i              /   @   s  d Z dZddlmZ ddlmZmZ ddlZddl	m
Z
 ddlmZ ddlZddlZddlZddlT ed	ejZed
ejZddlZejddddg Z[eeeeeeeeeee e!e"e#e$e%e&e'e(e)e*e+e,e-e.e/e0e1e2e3e4e5e6e7e8e9e:e;e<e=e>e?e@eAeBeCd.ZDG dd deEddZFdd ZGdd ZHdd ZIdZJdZKeJeHdeJ  eIeK ZLdZMdZNdZOdZPdZQeGeNeOePeQZRd ZSeGd!d"eIeS ZTd#eS ZUeGeTeUZVeGd$eVd% ZWeGeWeVeRZXd&d' ZYd(d) ZZeGeY  Z[d*Z\d+Z]d,Z^d-Z_eGe[d. e[d/ Z`eGe[d0 e[d1 ZaeGd2d3d4d5d6d7d8d9Zbd:ZceGd;d<d=ZdeGebecedZeeGeXeeeaeMZfeLef ZgeGe[d> eGd?d e[d@ eGdAd ZheGdBeKe`ZieJeGeieXeeeheM Zji Zkx@eY D ]6Zle\ekeld? < e]ekeldA < e^ekeld. < e_ekeld/ < qW em Znem Zox\eY D ]RZpx$epdA epd? fD ]Zqenreq qW x$epd/ epd. fD ]Zqeoreq q$W qW dCZsG dDdE dEetZuG dFdG dGetZvG dHdI dIZwdJd ZxdKdL ZydMd ZzdNdO ZdPd Z{dQdR Z|dSdT Z}dUdV Z~edWkre~  dS )XzKa-Ping Yee <ping@lfw.org>zpGvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, Skip Montanaro, Raymond Hettinger, Trent Nelson, Michael Foord    )open)lookupBOM_UTF8N)TextIOWrapper)chain)*z&^[ \t\f]*#.*?coding[:=][ \t]*([-\w.]+)s   ^[ \t\f]*(?:[#\r\n]|$)tokenizedetect_encoding
untokenize	TokenInfo).()[]:,;+-r   /|&<>=.%{}z==z!=z<=z>=~^z<<z>>z**z+=z-=z*=z/=z%=z&=z|=z^=z<<=z>>=z**=z//z//=z...z->@z@=c               @   s    e Zd Zdd Zedd ZdS )r   c             C   s$   d| j t| j  f }d| j|d S )Nz%d (%s)z8TokenInfo(type=%s, string=%r, start=%r, end=%r, line=%r))type)r"   tok_name_replace)selfZannotated_type r&   L/oak/stanford/groups/akundaje/marinovg/programs/Python-3.7.3/Lib/tokenize.py__repr___   s    zTokenInfo.__repr__c             C   s(   | j tkr| jtkrt| j S | j S d S )N)r"   OPstringEXACT_TOKEN_TYPES)r%   r&   r&   r'   
exact_typed   s    
zTokenInfo.exact_typeN)__name__
__module____qualname__r(   propertyr,   r&   r&   r&   r'   r   ^   s   ztype string start end linec              G   s   dd |  d S )Nr   r   r   )join)choicesr&   r&   r'   groupk       r3   c              G   s   t |  d S )Nr   )r3   )r2   r&   r&   r'   anyl   r4   r5   c              G   s   t |  d S )N?)r3   )r2   r&   r&   r'   maybem   r4   r7   z[ \f\t]*z	#[^\r\n]*z\\\r?\nz\w+z0[xX](?:_?[0-9a-fA-F])+z0[bB](?:_?[01])+z0[oO](?:_?[0-7])+z(?:0(?:_?0)*|[1-9](?:_?[0-9])*)z[eE][-+]?[0-9](?:_?[0-9])*z)[0-9](?:_?[0-9])*\.(?:[0-9](?:_?[0-9])*)?z\.[0-9](?:_?[0-9])*z[0-9](?:_?[0-9])*z[0-9](?:_?[0-9])*[jJ]z[jJ]c              C   sj   ddddddg} dh}xN| D ]F}x@t |D ]2}x,t jdd	 |D  D ]}|d| qFW q,W qW |S )
NbrufZbrZfr c             S   s   g | ]}||  fqS r&   )upper).0cr&   r&   r'   
<listcomp>   s    z(_all_string_prefixes.<locals>.<listcomp>)
_itertoolspermutationsproductaddr1   )Z_valid_string_prefixesresultprefixtr:   r&   r&   r'   _all_string_prefixes   s    
rH   c             C   s   t | t jS )N)recompileUNICODE)Zexprr&   r&   r'   _compile   s    rL   z[^'\\]*(?:\\.[^'\\]*)*'z[^"\\]*(?:\\.[^"\\]*)*"z%[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''z%[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""z'''z"""z'[^\n'\\]*(?:\\.[^\n'\\]*)*'z"[^\n"\\]*(?:\\.[^\n"\\]*)*"z\*\*=?z>>=?z<<=?z!=z//=?z->z[+\-*/%&@|^=<>]=?r   z[][(){}]z\r?\nz\.\.\.z[:;.,@]z'[^\n'\\]*(?:\\.[^\n'\\]*)*'z"[^\n"\\]*(?:\\.[^\n"\\]*)*"z
\\\r?\n|\Z   c               @   s   e Zd ZdS )
TokenErrorN)r-   r.   r/   r&   r&   r&   r'   rP      s    rP   c               @   s   e Zd ZdS )StopTokenizingN)r-   r.   r/   r&   r&   r&   r'   rQ      s    rQ   c               @   s,   e Zd Zdd Zdd Zdd Zdd Zd	S )
Untokenizerc             C   s   g | _ d| _d| _d | _d S )N   r   )tokensprev_rowprev_colencoding)r%   r&   r&   r'   __init__   s    zUntokenizer.__init__c             C   s   |\}}|| j k s&|| j kr>|| jk r>td||| j | j|| j  }|rb| jd|  d| _|| j }|r| jd|  d S )Nz+start ({},{}) precedes previous end ({},{})z\
r    )rU   rV   
ValueErrorformatrT   append)r%   startrowZcolZ
row_offsetZ
col_offsetr&   r&   r'   add_whitespace   s    

zUntokenizer.add_whitespacec             C   s4  t |}g }d}x|D ]
}t|dkr8| || P |\}}}}	}
|tkrV|| _q|tkr`P |tkrv|| qnl|tkr|	  |	\| _
| _qnL|ttfkrd}n:|r|r|d }|d t|kr| j| t|| _d}| | | j| |	\| _
| _|ttfkr|  j
d7  _
d| _qW d| jS )NF   TrS   r   r<   )iterlencompatENCODINGrW   	ENDMARKERINDENTr\   DEDENTpoprU   rV   NEWLINENLrT   r_   r1   )r%   iterableitindents	startlinerG   Ztok_typetokenr]   endlineindentr&   r&   r'   r
      sF    



zUntokenizer.untokenizec       
      C   s   g }| j j}|d ttfk}d}xt|g|D ]}|d d \}}	|tkrR|	| _q.|ttfkrf|	d7 }	|t	kr|rzd|	 }	d}nd}|t
kr||	 q.n>|tkr|  q.n*|ttfkrd}n|r|r||d  d}||	 q.W d S )Nr   Fr`   rY   Tra   )rT   r\   rj   rk   r   re   rW   NAMENUMBERSTRINGrg   rh   ri   )
r%   rp   rl   rn   Ztoks_appendro   Z
prevstringZtokZtoknumZtokvalr&   r&   r'   rd     s8    
zUntokenizer.compatN)r-   r.   r/   rX   r_   r
   rd   r&   r&   r&   r'   rR      s   %rR   c             C   s*   t  }|| }|jd k	r&||j}|S )N)rR   r
   rW   encode)rl   Zutoutr&   r&   r'   r
   8  s
    

c             C   sH   | d d   dd}|dks*|dr.dS |dks@|drDdS | S )	N   _r   zutf-8zutf-8-)zlatin-1z
iso-8859-1ziso-latin-1)zlatin-1-ziso-8859-1-ziso-latin-1-z
iso-8859-1)lowerreplace
startswith)Zorig_encZencr&   r&   r'   _get_normal_nameS  s    
r~   c                s   yj jW n tk
r$   d Y nX d d }d}fdd} fdd}| }|trpd |dd  }d	}|s||g fS ||}|r||gfS t|s||gfS | }|s||gfS ||}|r|||gfS |||gfS )
NFzutf-8c                  s    y  S  t k
r   dS X d S )Nr4   )StopIterationr&   )readliner&   r'   read_or_stopv  s    z%detect_encoding.<locals>.read_or_stopc                s   y|  d}W n4 tk
rB   d}d k	r6d|}t|Y nX t|}|sVd S t|d}yt|}W n: t	k
r   d krd| }nd|}t|Y nX  r|dkr؈d krd}n
d}t||d	7 }|S )
Nzutf-8z'invalid or missing encoding declarationz{} for {!r}rS   zunknown encoding: zunknown encoding for {!r}: {}zencoding problem: utf-8z encoding problem for {!r}: utf-8z-sig)
decodeUnicodeDecodeErrorr[   SyntaxError	cookie_rematchr~   r3   r   LookupError)rr   Zline_stringmsgr   rW   Zcodec)	bom_foundfilenamer&   r'   find_cookie|  s6    


z$detect_encoding.<locals>.find_cookieT   z	utf-8-sig)__self__nameAttributeErrorr}   r   blank_rer   )r   rW   defaultr   r   firstsecondr&   )r   r   r   r'   r	   ^  s8    
&




c             C   sV   t | d}y0t|j\}}|d t||dd}d|_|S    |   Y nX d S )Nrbr   T)line_bufferingr9   )_builtin_openr	   r   seekr   modeclose)r   bufferrW   linestextr&   r&   r'   r     s    

r   c             C   sB   ddl m}m} t| \}}t| d}|d}t||||j|S )Nr   )r   repeatr4   )	itertoolsr   r   r	   rb   	_tokenize__next__)r   r   r   rW   consumedZrl_genemptyr&   r&   r'   r     s
    
c             c   s  d } }}d}d\}}d }dg}	|d k	rH|dkr6d}t t|dddV  d}
d}xy|}
|  }W n tk
rz   d}Y nX |d k	r||}|d	7 }dt| }}|rt|std
|||}|r
|d }}t t||d |  |||f|| V  d\}}d }nf|r^|dd  dkr^|dd  dkr^t t	|| ||t|f|V  d}d }qTn|| }|| }qTn|dkr|s|sP d}xf||k r|| dkr|d	7 }n6|| dkr|t
 d	 t
 }n|| dkrd}nP |d	7 }qW ||krP || dkr|| dkr`||d  d}t t|||f||t| f|V  |t|7 }t t||d  ||f|t|f|V  qT||	d kr|	| t t|d | |df||f|V  xp||	d k r||	krtdd|||f|	d d }	t td||f||f|V  qW n|s0td|dfd}x||k rtt||}|r|d	\}}||f||f|  }}}||krq8||| ||  }}||ks|dkr|dkr|dkrt t||||V  q|dkr|dkrt t||||V  nt t||||V  q|dkr>t t||||V  q|tkrtt| }|||}|r|d}||| }t t||||f|V  n||f}||d  }|}P q|tks|d d tks|d d tkrR|d dkr>||f}tt|p t|d	 p t|d }||d  d	 }}|}P nt t||||V  nf| rpt t||||V  nH|dkrd	}n8|dkr|d	7 }n|d kr|d	8 }t t||||V  n*t t	|| ||f||d	 f|V  |d	7 }q8W qTW |
r0|
d dkr0t td|d	 t|
f|d	 t|
d	 fdV  x0|	d	d  D ] }t td|df|dfdV  q>W t td|df|dfdV  d S )!Nr   Z
0123456789)r<   r   z	utf-8-sigzutf-8)r   r   r<   r4   rS   zEOF in multi-line stringz\
z\
rY   	z#
#z
ra   z3unindent does not match any outer indentation levelz
<tokenize>zEOF in multi-line statementr   z...r`   r   
\z([{z)]})r   re   r   r   rc   rP   r   rq   rv   Z
ERRORTOKENtabsizerstripZCOMMENTrk   r\   rg   IndentationErrorrh   rL   PseudoTokenspanru   rj   triple_quotedendpatssingle_quotedgetisidentifierrt   r)   rf   )r   rW   ZlnumZparenlevZ	continuedZnumcharsZcontstrZneedcontZcontlinern   	last_linerr   posmaxZstrstartZendprogZendmatchrq   columnZcomment_tokenZpseudomatchr]   ZsposZeposrp   Zinitialrs   r&   r&   r'   r     s   




*
 


"
"













. r   c             C   s
   t | d S )N)r   )r   r&   r&   r'   generate_tokens  s    r   c           
      s(  dd l } dd  d fdd	}| jdd}|jdd	d
dd |jdddddd | }y|jr|j}t|d}tt|j}W d Q R X nd}t	t
jjd }xF|D ]>}|j}|jr|j}d|j|j  }	td|	t| |jf  qW W n8 tk
r: }
 z0|
jd dd \}}||
jd |||f W d d }
~
X Y n tk
r }
 z(|
jd \}}||
jd |||f W d d }
~
X Y n tk
r }
 z||
| W d d }
~
X Y nx tk
r }
 z||
 W d d }
~
X Y nN tk
r   td Y n2 tk
r" }
 z d|
   W d d }
~
X Y nX d S )Nr   c             S   s   t | tjd d S )N)file)printsysstderr)messager&   r&   r'   perror  s    zmain.<locals>.perrorc                sR   |r"|f| | f } d|  n"|r8 d|| f  n d|   t d d S )Nz%s:%d:%d: error: %sz%s: error: %sz	error: %srS   )r   exit)r   r   locationargs)r   r&   r'   error  s    zmain.<locals>.errorzpython -m tokenize)Zprogr   r6   zfilename.pyz'the file to tokenize; defaults to stdin)destZnargsZmetavarhelpz-ez--exactexactZ
store_truez(display token names using the exact type)r   actionr   r   z<stdin>z%d,%d-%d,%d:z%-20s%-15s%-15rrS   r   zinterrupted
zunexpected error: %s)NN)argparseZArgumentParserZadd_argumentZ
parse_argsr   r   listr   r   r   r   stdinr"   r   r,   r]   rq   r   r#   r*   r   r   rP   r   OSErrorKeyboardInterrupt	Exception)r   r   Zparserr   r   r;   rT   rp   Z
token_typeZtoken_rangeerrrr   r   r&   )r   r'   main  sN    
&&r   __main__)
__author__Z__credits__builtinsr   r   codecsr   r   collectionsior   r   r   rA   rI   r   rp   rJ   ASCIIr   r   __all__ZLPARZRPARZLSQBZRSQBZCOLONZCOMMAZSEMIZPLUSZMINUSZSTARZSLASHZVBARZAMPERZLESSZGREATERZEQUALZDOTZPERCENTZLBRACEZRBRACEZEQEQUALZNOTEQUALZ	LESSEQUALZGREATEREQUALZTILDEZ
CIRCUMFLEXZ	LEFTSHIFTZ
RIGHTSHIFTZ
DOUBLESTARZ	PLUSEQUALZMINEQUALZ	STAREQUALZ
SLASHEQUALZPERCENTEQUALZ
AMPEREQUALZ	VBAREQUALZCIRCUMFLEXEQUALZLEFTSHIFTEQUALZRIGHTSHIFTEQUALZDOUBLESTAREQUALZDOUBLESLASHZDOUBLESLASHEQUALZELLIPSISZRARROWATZATEQUALr+   
namedtupler   r3   r5   r7   Z
WhitespaceZCommentZIgnoreZNameZ	HexnumberZ	BinnumberZ	OctnumberZ	DecnumberZ	IntnumberZExponentZ
PointfloatZExpfloatZFloatnumberZ
ImagnumberZNumberrH   rL   ZStringPrefixZSingleZDoubleZSingle3ZDouble3ZTripleZStringZOperatorZBracketZSpecialZFunnyZ
PlainTokenZTokenZContStrZPseudoExtrasr   r   Z_prefixsetr   r   rG   r:   rD   r   r   rP   rQ   rR   r
   r~   r	   r   r   r   r   r-   r&   r&   r&   r'   <module>   s  





_] :<
