
NV]c           @@  s  d  Z  d d l m Z d d l Z d d l Z d d l Z d d l m Z d d l Z	 d d l
 m Z d d l m Z d d l m Z d d l m Z d	 Z e j Z e j Z e j Z e j Z e j Z e j Z e j Z e j Z e j Z e j Z e j Z e j Z e j Z e d
 d  Z  e j! d  Z" i  Z# e d k rMe$ j% Z& n	 d   Z& d   Z' d   Z( e) e) d  Z* d   Z+ d   Z, e+ d  Z- e+ d  Z. e+ d  Z/ e+ d  Z0 d   Z1 d e d d d d d g  f d     YZ2 d e2 f d      YZ3 d! e4 f d"     YZ5 d#   Z6 d$   Z7 d- d&  Z8 d'   Z9 d. d(  Z: d)   Z; e< d* k re= e j>  d+ k re j> d% Z? e@ e?   ZA eA jB   ZC Wd QXn e jD jB   ZC d d, l mE ZE mF ZF eG eC eH  reE eC  ZC n  x" e8 eC eF    D] ZI eI GHqWn  d S(/   s  
This tokenizer has been copied from the ``tokenize.py`` standard library
tokenizer. The reason was simple: The standard library tokenizer fails
if the indentation is not right. To make it possible to do error recovery the
    tokenizer needed to be rewritten.

Basically this is a stripped down version of the standard library module, so
you can read the documentation there. Additionally we included some speed and
memory optimizations here.
i    (   t   absolute_importN(   t
   namedtuple(   t   BOM_UTF8(   t   PythonTokenTypes(   t
   py_version(   t   split_liness
   \U0010fffft   TokenCollectionsc   pseudo_token single_quoted triple_quoted endpats whitespace fstring_pattern_map always_break_tokenss   utf-8i   c         C@  s   t  S(   N(   t   True(   t   s(    (    s4   lib/python2.7/site-packages/parso/python/tokenize.pyt   is_identifier>   s    c          O@  sM   | j  d t  } | s t  d } | s8 | d 7} n  | d j |   d S(   Nt   capturet   (s   ?:t   |t   )(   t   popt   Falset   AssertionErrort   join(   t   choicest   kwargsR
   t   start(    (    s4   lib/python2.7/site-packages/parso/python/tokenize.pyt   groupB   s    c          G@  s   t  |    d S(   Nt   ?(   R   (   R   (    (    s4   lib/python2.7/site-packages/parso/python/tokenize.pyt   maybeL   s    c   	      C@  s
  d   } d d d g } |  d k r4 | j  d  n  t d g  } |  d k r | r d
 d g } | ry | } t   } q | | 7} n | r t   Sx; | D]3 } x* t j |  D] } | j | |   q Wq W|  d k r| j | d   | j | d   n  | S(   Nc         s@  sK   xD t  j g  |  D] } | | j   f ^ q   D] } d j |  Vq/ Wd  S(   Nt    (   t
   _itertoolst   productt   upperR   (   t   prefixt   cR   (    (    s4   lib/python2.7/site-packages/parso/python/tokenize.pyt   different_case_versionsR   s    5t   bt   rt   ui   i    t   brR   i   t   ft   fri   i   t   ur(   i   i    (   i   i   (   i   i   (   t   appendt   setR   t   permutationst   update(	   t   version_infot   include_fstringt   only_fstringR   t   valid_string_prefixest   resultR#   R   t   t(    (    s4   lib/python2.7/site-packages/parso/python/tokenize.pyt   _all_string_prefixesQ   s(    	c         C@  s   t  j |  t  j  S(   N(   t   ret   compilet   UNICODE(   t   expr(    (    s4   lib/python2.7/site-packages/parso/python/tokenize.pyt   _compilet   s    c         C@  sE   y t  t |   SWn, t k
 r@ t |   t  t |   <} | SXd  S(   N(   t   _token_collection_cachet   tuplet   KeyErrort   _create_token_collection(   R*   R.   (    (    s4   lib/python2.7/site-packages/parso/python/tokenize.pyt   _get_token_collectionx   s
    s'   (?:\{\{|\}\}|\\(?:\r\n?|\n)|[^{}\r\n])+s   (?:[^{}]+|\{\{|\}\})+s   (?:\\(?:\r\n?|\n)|[^{}\r\n])+s   [^{}]+c   -   
   C@  s  d } t  |  } d } |  d d k r1 d } n* t j d d k rM d } n d t d } |  dH k r d } d } d } d } t | | | |  }	 d }
 t d d  t |
  } d |
 } t | |  } t d | d  } n d } d } |  dI k rd } n d } d } t | | | |  }	 |  d d	 k  rA|	 d 7}	 n  d }
 t d d  t |
  } d |
 } t | |  } t d | d  } t | | |	  } t |   } t |   } t t |  d  t   } t |  d  t d! t } t |   } d" } d# } d$ } d% } t | d& | d'  } t d( d) d* d+ d, d- d.  } d/ } d0 d1 d2 g } |  dJ k rs| j d d3  n  |  dK k r| j d d5  n | j d d6  t |   } t | | |  } t | d7 t d8 d9  | d: t d; d9   } | | g }  dL }! | r#|  j	 | t |!    n  t d< |   }" t | d= t t |" | | | | d= t }# i  }$ x^ | D]V }% t  |  |$ |% d8 <t  |  |$ |% d; <t  |  |$ |% d& <t  |  |$ |% d' <qmWt
   }& t
   }' i  }( xR | D]J }) x dM D] }* |& j |) |*  qWx dN D] }* |' j |) |*  qWqWx- | D]% }) x |! D] }* |* |( |) |* <qHWq;WdO }+ t  |#  }, t |, |& |' |$ | |( |+  S(P   Ns   [ \f\t]*s	   #[^\r\n]*i    i   s   ([A-Za-z_0-9]+)s   (\w+)u   ([A-Za-z_0-9-s   ]+)i   i   s   0[xX](?:_?[0-9a-fA-F])+s   0[bB](?:_?[01])+s   0[oO](?:_?[0-7])+s   (?:0(?:_?0)*|[1-9](?:_?[0-9])*)s   [eE][-+]?[0-9](?:_?[0-9])*s)   [0-9](?:_?[0-9])*\.(?:[0-9](?:_?[0-9])*)?s   \.[0-9](?:_?[0-9])*s   [0-9](?:_?[0-9])*s   [0-9](?:_?[0-9])*[jJ]s   [jJ]s   0[xX][0-9a-fA-F]+s
   0[bB][01]+s   0[oO][0-7]+s   0[oO]?[0-7]+s   (?:0+|[1-9][0-9]*)s   [lL]?s   [eE][-+]?[0-9]+s   [0-9]+\.[0-9]*s   \.[0-9]+s   [0-9]+s
   [0-9]+[jJ]R+   R,   s   (?:\\.|[^'\\])*'s   (?:\\.|[^"\\])*"s   (?:\\.|'(?!'')|[^'\\])*'''s   (?:\\.|"(?!"")|[^"\\])*"""s   '''s   """s   \*\*=?s   >>=?s   <<=?s   //=?s   ->s   [+\-*/%&@`|^!=<>]=?t   ~s   [][(){}]s   \r\n?s   \ns   [;.,@]s   \.\.\.i   s   :=?t   :s   '[^\r\n'\\]*(?:\\.[^\r\n'\\]*)*t   's   \\(?:\r\n?|\n)s   "[^\r\n"\\]*(?:\\.[^\r\n"\\]*)*t   "s   \\(?:\r\n?|\n)|\ZR
   t   ;t   importt   classt   deft   tryt   exceptt   finallyt   whilet   witht   return(   i   i   (   i   i    (   i   i    (   i   i   (   R>   R=   s   """s   '''(   R>   R=   (   s   """s   '''(
   R?   R@   RA   RB   RC   RD   RE   RF   RG   RH   (   R5   t   sysR*   t   MAX_UNICODER   R   R0   R   t   insertR&   R'   t   addR   (-   R*   t
   Whitespacet
   whitespacet   Commentt   Namet	   Hexnumbert	   Binnumbert	   Octnumbert	   Decnumbert	   Intnumbert   Exponentt
   Pointfloatt   Expfloatt   Floatnumbert
   Imagnumbert   Numbert   possible_prefixest   StringPrefixt   StringPrefixWithFt   fstring_prefixest   FStringStartt   Singlet   Doublet   Single3t   Double3t   Triplet   Operatort   Brackett   special_argst   Specialt   Funnyt   ContStrt   pseudo_extra_poolt
   all_quotest   PseudoExtrast   PseudoTokent   endpatst   _prefixt   single_quotedt   triple_quotedt   fstring_pattern_mapR/   t   quotet   ALWAYS_BREAK_TOKENSt   pseudo_token_compiled(    (    s4   lib/python2.7/site-packages/parso/python/tokenize.pyR9      s    		
	
	
		 t   Tokent   typet   stringt	   start_posR   c           B@  s   e  Z e d     Z RS(   c         C@  si   t  |  j  } t |  d k r@ |  j d t |  d d f S|  j d |  j d t |  j  f Sd  S(   Ni   i    (   R   Rz   t   lenR{   (   t   selft   lines(    (    s4   lib/python2.7/site-packages/parso/python/tokenize.pyt   end_pos  s    (   t   __name__t
   __module__t   propertyR   (    (    (    s4   lib/python2.7/site-packages/parso/python/tokenize.pyRx     s   t   PythonTokenc           B@  s   e  Z d    Z RS(   c         C@  s   d |  j  d |  j j  S(   Ns6   TokenInfo(type=%s, string=%r, start_pos=%r, prefix=%r)Ry   (   t   _replaceRy   t   name(   R}   (    (    s4   lib/python2.7/site-packages/parso/python/tokenize.pyt   __repr__  s    (   R   R   R   (    (    (    s4   lib/python2.7/site-packages/parso/python/tokenize.pyR     s   t   FStringNodec           B@  s>   e  Z d    Z d   Z d   Z d   Z d   Z d   Z RS(   c         C@  s1   | |  _  d |  _ d |  _ d  |  _ d |  _ d  S(   Ni    R   (   Ru   t   parentheses_countt   previous_linest   Nonet   last_string_start_post   format_spec_count(   R}   Ru   (    (    s4   lib/python2.7/site-packages/parso/python/tokenize.pyt   __init__  s
    				c         C@  s   |  j  d 7_  d  S(   Ni   (   R   (   R}   t	   character(    (    s4   lib/python2.7/site-packages/parso/python/tokenize.pyt   open_parentheses(  s    c         C@  s.   |  j  d 8_  |  j  d k r* d |  _ n  d  S(   Ni   i    (   R   R   (   R}   R   (    (    s4   lib/python2.7/site-packages/parso/python/tokenize.pyt   close_parentheses+  s    c         C@  s   t  |  j  d k S(   Ni   (   R|   Ru   (   R}   (    (    s4   lib/python2.7/site-packages/parso/python/tokenize.pyt   allow_multiline1  s    c         C@  s   |  j  |  j k S(   N(   R   R   (   R}   (    (    s4   lib/python2.7/site-packages/parso/python/tokenize.pyt
   is_in_expr4  s    c         C@  s   |  j    o |  j S(   N(   R   R   (   R}   (    (    s4   lib/python2.7/site-packages/parso/python/tokenize.pyt   is_in_format_spec7  s    (   R   R   R   R   R   R   R   R   (    (    (    s4   lib/python2.7/site-packages/parso/python/tokenize.pyR     s   						c         C@  s   xz t  |   D]l \ } } | j | j  r t t | j | d | } d } | j s\ t  |  | 3| d t | j  f Sq Wd  | d f S(   NR   R   i    (	   t	   enumeratet
   startswithRu   R   t   FSTRING_ENDR   R   R|   R   (   t   fstring_stackRz   R{   t   additional_prefixt   fstring_stack_indext   nodet   token(    (    s4   lib/python2.7/site-packages/parso/python/tokenize.pyt   _close_fstring_if_necessary;  s    	c         C@  sZ  | d } | j    } | j   r: | r1 t } qO t } n | rI t } n t } | j | |  } | d  k rz | j | f S| j s | | f | _	 n  | j
 d  }	 xP | D]H }
 |  |
 j j |	  } | d  k	 r | j
 d  t |
 j   }	 q q W| } | t |	  7} |	 j d  s+|	 j d  rC| j |	 7_ d }	 n | j |	 }	 |	 | f S(   Nii    s   
s   R   (   R   R   t   fstring_format_spec_multi_linet   fstring_format_spec_single_linet   fstring_string_multi_linet   fstring_string_single_linet   matchR   R   R   R   Ru   R|   t   endswith(   Rp   R   t   linet   lnumt   post   tosR   t   regexR   Rz   t   fstring_stack_nodet	   end_matcht   new_pos(    (    s4   lib/python2.7/site-packages/parso/python/tokenize.pyt   _find_fstring_stringK  s4    
				$	i   c         C@  s%   t  |  d t } t | | d | S(   s0   Generate tokens from a the source code (string).t   keependsR{   (   R   R   t   tokenize_lines(   t   codeR*   R{   R~   (    (    s4   lib/python2.7/site-packages/parso/python/tokenize.pyt   tokenizes  s    c         @  s     f d   } | S(   sL   
    A small helper function to help debug the tokenize_lines function.
    c          ?@  s#   x   |  |   D] } | Vq Wd  S(   N(    (   t   argsR   R   (   t   func(    s4   lib/python2.7/site-packages/parso/python/tokenize.pyt   wrapper}  s    (    (   R   R   (    (   R   s4   lib/python2.7/site-packages/parso/python/tokenize.pyt   _print_tokensy  s    c   )      #@  s      f d   } t  |  \ } } } } } }	 }
 d } d g   d } d } d } d } t } d } d } t } | d d  g  } x|  D]}  d 7 d } t |  } | r| j t  r t } | d } t |  } n  d | d | } | d } | | d 7} t } n  | r| j |  } | rn| j d  } t	 t
 | | |  | |  Vd } d } q| | } | | } q n  x| | k  rc| re| d } | j   st | | |  |  \ } } | r t	 t | | j d d Vd | _ qn  | | k rPqn  | | } t | |  | f |  \ } } } | | 7} | d k	 re| Vqqen  | j | |  } | s| j | |  }  | d k rx" | |  j    D] }! |! VqWn  |  j   } t } t	 t | |  | f | |  j d   Vd } | d 7} qn  | | j d  } d } | j d	  \ }" }  |" f  | j d	  }# |# d k r~| stt  | } Pn  |# d }$ | r@|$ d
 k r@t } | d k r@| r@d }% |" }& x( | |% d k r|% d 7}% |& d 8}& qW|&   d k rt	 t d  d  V  j |&  n  x | |&  D] }! |! Vq+Wq@n  |$ | k sp|$ d k r|# d k r|# d k rt	 t |#  |  Vq| j d  d k	 rc|# |
 k rg  | (d } t j d | |"   rxJ t r  j   }' |' |" k rt	 t d  d  Vq  j |'  PqWqn  t |#  r>t	 t |#  |  Vq`xt |#  |  D] }! |! VqQWq|$ d k rt d   | D  rg  | (n  | r| d k r| rt	 t |#  |  Vn
 | |# } t } q|$ d k r|# j  d  st  | |# } q|# | k r| |# } | j | |  } | rc| j d  } | |" | !}# t	 t
 |#  |  Vq` |" f } | |" } | } Pq|$ | k s|# d	  | k s|# d  | k r+|# d d k r |" f } | j! |$  p| j! |# d  p| j! |# d	  } | |" } | } Pq`t	 t
 |#  |  Vq|# |	 k re| j t" |	 |#   t	 t# |#  |  Vq|$ d k r| |" d k r| | | |" 7} Pq|# d k r| r| d j$ |#  qL| d 7} n |# d k r| r| d j% |#  qL| rL| d 8} qLnF |# d k rL| rL| d j& | d j' d k rL| d j' d 7_' n  t	 t( |#  |  VqWq W| rt	 t | | |  V| j  d  s| j  d  rt } qn   | f }( x&   d D] }' t	 t d |( d  VqWt	 t) d |( |  Vd S(   s)  
    A heavily modified Python standard library tokenizer.

    Additionally to the default information, yields also the prefix of each
    token. This idea comes from lib2to3. The prefix contains all information
    that is irrelevant for the parser like newlines in parentheses or comments.
    c         3@  sg   x` |    d k  rb |    d k rA t  t d  d f d  VPn  t  t d  d  V  j   q Wd  S(   NiiR   i    (   R   t   ERROR_DEDENTt   DEDENTR   (   R   (   t   indentsR   t   spos(    s4   lib/python2.7/site-packages/parso/python/tokenize.pyt   dedent_if_necessary  s    i    t
   0123456789R   i   t   ^iR   i   s   
\#s   t   .s   ...i   s	   [ \f\t]*$s   
c         s@  s   |  ] } | j    Vq d  S(   N(   R   (   t   .0R#   (    (    s4   lib/python2.7/site-packages/parso/python/tokenize.pys	   <genexpr>'  s    t   #s   
s   \s   \
s   \
s   \s   ([{s   )]}R<   s   N(   s   \
s   \
s   \(*   R:   R   R   R|   R   t   BOM_UTF8_STRINGR   R   t   endR   t   STRINGR   R   t   FSTRING_STRINGR   R   R   t
   ERRORTOKENR   t   spanR   t   INDENTR&   t   NUMBERR1   R   R   R	   t   NAMEt   _split_illegal_unicode_namet   anyt   NEWLINER   t   getR   t   FSTRING_STARTR   R   R   R   t   OPt	   ENDMARKER()   R~   R*   R{   R   t   pseudo_tokenRr   Rs   Rp   RN   Rt   t   always_break_tokenst   paren_levelt   maxt   numcharst   contstrt   contlinet   new_lineR   R   t   firstR   R   R   t   endprogt   endmatcht   contstr_startR   Rz   t   restt   fstring_end_tokent   quote_lengtht   pseudomatchR   R/   R   R   t   initialt   it   indent_startt   indentR   (    (   R   R   R   s4   lib/python2.7/site-packages/parso/python/tokenize.pyR     sd   
!	


		


		
	
	


$	

	

"

c         #@  s
       f d   } d   t   |  x t |   D] \ } }  r t |  r |   V|   t   d  | d | d | f  q   | 7  q7   | } t |  r |   q7   r |   Vd  | d | d | f  n  |   t  q7 W  r|   Vn  d  S(   Nc           @  s   t   r t n t      S(   N(   R   R   R   (    (   t   foundt
   is_illegalR   R   (    s4   lib/python2.7/site-packages/parso/python/tokenize.pyt   create_token  s    R   i    i   (   R   R   R	   R   (   R   R{   R   R   R   t   chart	   new_found(    (   R   R   R   R   s4   lib/python2.7/site-packages/parso/python/tokenize.pyR     s0    
	
t   __main__i   (   t   python_bytes_to_unicodet   parse_version_string(   i   i    (   i   i    (J   t   __doc__t
   __future__R    RI   Rz   R1   t   collectionsR   t	   itertoolsR   t   codecsR   t   parso.python.tokenR   t   parso._compatibilityR   t   parso.utilsR   RJ   R   R   R   R   R   R   R   R   R   R   R   R   R   R   t   decodeR   R6   t   strt   isidentifierR	   R   R   R   R0   R5   R:   R   R   R   R   R9   Rx   R   t   objectR   R   R   R   R   R   R   R   R|   t   argvt   patht   openR#   t   readR   t   stdinR   R   t
   isinstancet   bytesR   (    (    (    s4   lib/python2.7/site-packages/parso/python/tokenize.pyt   <module>   s|   																
	#				+
		(		!