B
    t\%j                 @   s  d Z ddlZddlZddlmZ ddlZddlmZ ddlm	Z	 ddl
mZ ddlmZ ddlmZ eeZd	Zd
Zdd Zdd Zdd Zdd Zdd Zdd Zd,ddZdd Zdd Zdd Zd d! ZG d"d# d#e Z!d$d% Z"G d&d' d'e#Z$G d(d) d)e#Z%G d*d+ d+e#Z&dS )-z
Basically a contains parser that is faster, because it tries to parse only
parts and if anything changes, it only reparses the changed parts.

It works with a simple diff in the beginning and will try to reuse old parser
fragments.
    N)
namedtuple)split_lines)Parser)	EndMarker)PythonToken)PythonTokenTypesF)INDENTERROR_DEDENTDEDENTc             C   s*   x$| r$| j dkr$| jtkr$|  } qW | S )N
error_leaf)type
token_type_INDENTATION_TOKENSget_previous_leaf)leaf r   0lib/python3.7/site-packages/parso/python/diff.py!_get_previous_leaf_if_indentation   s    
r   c             C   s*   x$| r$| j dkr$| jtkr$|  } qW | S )Nr   )r   r   r   r   )r   r   r   r   _get_next_leaf_if_indentation    s    
r   c       	      C   s<  y
| j }W n tk
r   | jdkrH| jtkrH| jr:t| jrDtdS t| 	 }|dkrh| j}d}n*|j
| jkst|| f|j| j }|j}d|ksd|krt|}|d t| d }|t|d f}n|d |d t| f}| j|kst| j|fY n2X x.|D ]&}|j| ks*t| |ft| qW dS )	z~
    Checks if the parent/children relationship is correct.

    This is a check that only runs during debugging/testing.
    r   N)   r   
r   r   )childrenAttributeErrorr   r   r   valueAssertionErrorprefixr   r   end_pos	start_posr   lenparent_assert_valid_graph)	noder   Zprevious_leafZcontentZprevious_start_posZsplittedlineZactualZchildr   r   r   r"   '   s0    



 
r"   c             C   sL   t |  dd}t||}t||}dd l}d|jd|d|f S )NT)keependsr   zmThere's an issue with the diff parser. Please report (parso v%s) - Old/New:
%s
Actual Diff (May be empty):
%s )r   get_codedifflibZunified_diffparso__version__join)module	old_lines	new_linesZcurrent_linesZcurrent_diffZold_new_diffr)   r   r   r   _get_debug_error_messageQ   s    r/   c             C   s(   |   }t|r|jd S |jd S d S )Nr   )get_last_leaf_ends_with_newliner   r   )Znode_or_leaf	last_leafr   r   r   _get_last_line]   s    
r3   c             C   s.   x(| d k	r(| j dkr(| jdkr(|  } qW | S )Nr   r
   )r   r   r   )r   r   r   r   _skip_dedent_error_leavese   s    r4   r&   c             C   s@   t | } | jdkr| j }n| j}|dkp>|dp>|dS )Nr   newliner   r   )r4   r   r   lowerendswith)r   suffixtypr   r   r   r1   k   s
    
r1   c             C   s    x|D ]}|j dkrdS qW dS )zg
    if, while, for and try might not be finished, because another part might
    still be parsed.
    )Zif_stmtZ
while_stmtZfor_stmtZtry_stmtFT)nonterminal)pgen_grammarstack
stack_noder   r   r   _flows_finishedv   s    

r>   c             C   sB   | j dkr| jd } | j dkr(| jd } | j dko@| jd j dkS )N	decoratedr   )async_funcdef
async_stmt)ZclassdefZfuncdefsuite)r   r   )r#   r   r   r   _func_or_class_has_suite   s
    



rC   c             C   sJ   t | |sdS x6t|D ]*}|jdkr*dS |jdkrt|jdkS qW dS )NFZ	decoratorrB   r   T)r>   reversedr:   r    nodes)r;   r<   r=   r   r   r   _suite_or_file_input_is_valid   s    


rF   c             C   sB   | j dkr| jd } y| jd j}W n tk
r8   dS X |dkS )NrA   r   r   F)ifforwhiletrywith)r   r   r   r   )r#   r   r   r   r   _is_flow_node   s    

rL   c               @   s   e Zd ZdS )_PositionUpdatingFinishedN)__name__
__module____qualname__r   r   r   r   rM      s   rM   c          	   C   sX   xR| D ]J}y
|j }W n. tk
rB   | j|7  _||kr>tY qX t||| qW d S )N)r   r   r$   rM   _update_positions)rE   line_offsetr2   r#   r   r   r   r   rQ      s    


rQ   c               @   sZ   e Zd ZdZdd Zdd Zdd Zdd	 Zd
d Zdd Z	dd Z
dd ZdddZdS )
DiffParserz
    An advanced form of parsing a file faster. Unfortunately comes with huge
    side effects. It changes the given module.
    c             C   s   || _ || _|| _d S )N)_pgen_grammar
_tokenizer_module)selfr;   Z	tokenizerr,   r   r   r   __init__   s    zDiffParser.__init__c             C   s   d| _ d| _t| j| _d S )Nr   )_copy_count_parser_count
_NodesTreerV   _nodes_tree)rW   r   r   r   _reset   s    zDiffParser._resetc          	   C   s  t d d| j_|| _|   t|}td|| j}|	 }t dt||f  x|D ]\}}}}	}
t d||d ||	d |
 |
|kr|d dkr|
d8 }
|dkr|	| }| 
|||
 q\|d	kr| j|
d
 q\|dkr| j|
d
 q\|dks\tq\W | j  trby(| j d|ks*tt| j W n* tk
r`   tt| j||  Y nX | jjd }||krtd||f t| j|| t d | jS )a  
        The algorithm works as follows:

        Equal:
            - Assure that the start is a newline, otherwise parse until we get
              one.
            - Copy from parsed_until_line + 1 to max(i2 + 1)
            - Make sure that the indentation is correct (e.g. add DEDENT)
            - Add old and change positions
        Insert:
            - Parse from parsed_until_line + 1 to min(j2 + 1), hopefully not
              much more.

        Returns the new module node.
        zdiff parser startNzline_lengths old: %s; new: %sz!-> code[%s] old[%s:%s] new[%s:%s]r   r   r&   Zequalreplace)
until_lineinsertdeleter   z(%s != %s) zdiff parser end)LOGdebugrV   Z_used_names_parser_lines_newr]   r    r(   ZSequenceMatcherZget_opcodes_copy_from_old_parser_parser   r\   closeDEBUG_DIFF_PARSERr'   r+   r"   printr/   r   	Exception)rW   r-   r.   Zline_lengthZsmZopcodesZ	operationZi1Zi2Zj1Zj2rR   Zlast_posr   r   r   update   sH    




zDiffParser.updatec             C   s2   | j  d|kr.tdd|d| d S )Nr&   zparser issue:
%s
%s)rV   r'   r+   rb   Zwarning)rW   r-   Z	lines_newr   r   r   _enabled_debugging  s    zDiffParser._enabled_debuggingc             C   s   d}x|| j jkr| j j| }| |d }|d krH| | j jd  n|jj}||}| j jd }	| j ||d  ||}
|
r|  jd7  _| j j}t	
d|
d jd |
d jd d |	| n| | j jd  || j jkst|| j j}qW d S )Nr   r   zcopy old[%s:%s] new[%s:%s]r   )r\   parsed_until_line_get_old_line_stmtrf   r!   r   index
copy_nodesrY   rb   rc   r   r   r   )rW   rR   Zuntil_line_oldZuntil_line_newlast_until_lineZparsed_until_line_oldZ	line_stmtZ
p_childrenro   Zfrom_Zcopied_nodestor   r   r   re     s,    

z DiffParser._copy_from_old_parserc             C   sf   | j j|dfdd}t|r$| }| d |krb|}x|jjdkrN|j}q:W |jd |krb|S d S )Nr   T)Zinclude_prefixes)
file_inputrB   )rV   Zget_leaf_for_positionr1   get_next_leafget_start_pos_of_prefixr!   r   r   )rW   Zold_liner   r#   r   r   r   rn   5  s    
zDiffParser._get_old_line_stmtc             C   s|   d}xr|| j jkrv| |}|j}| j | td|d  d | j j|jd d  || j jkslt	|| j j}qW dS )zy
        Parses at least until the given line, but might just parse more until a
        valid state is reached.
        r   z/parse_part from %s to %s (to %s in part parser)r   N)
r\   rm   _try_parse_partr   add_parsed_nodesrb   rc   ru   r   r   )rW   r_   rq   r#   rE   r   r   r   rf   F  s    
zDiffParser._parsec             C   sR   |  j d7  _ | jj}| j|d }| j|||d}t| jdd| _| jj|dS )z
        Sets up a normal parser that uses a spezialized tokenizer to only parse
        until a certain position (or a bit longer if the statement hasn't
        ended.
        r   N)rR   T)Zerror_recovery)tokens)	rZ   r\   rm   rd   _diff_tokenizer   rT   _active_parserparse)rW   r_   rm   Zlines_afterrx   r   r   r   rv   ]  s    
zDiffParser._try_parse_partr   c             c   s  d}d}g }|  |d}| jj}x|D ]\}	}
}}|d | |d f}|	tjkrp||d  |rpd}d}q(d}|	tjks|	tjkr:|r:t|dkr:|	  |r|st
|\}	}
}}d|ksd|krtdd	|}n6|d t|kstt||d t| dkrd	}ttjd	|d | df|V  P n|	tjkr|d |krt|	|
||V  t| j|r(|d d df}x2t|t|kr|	  ttjd	|d	V  qW ttjd	|d	V  P nq(t|	|
||V  q(W d S )
NTF)r   r   r   r   r   r   z
[^\n\r]+\Zr&   )rU   rz   r<   r   r   appendr
   r	   r    popnextresubr   reprr   	ENDMARKERNEWLINErF   rT   int)rW   linesr_   rR   Zis_first_tokenZomitted_first_indentindentsrx   r<   r9   stringr   r   r   r   r   ry   s  sR    

zDiffParser._diff_tokenizeN)r   )rN   rO   rP   __doc__rX   r]   rk   rl   re   rn   rf   rv   ry   r   r   r   r   rS      s   F&rS   c               @   sB   e Zd ZeddZdddZdd Zdd	 ZdddZdd Z	dS )_NodesTreeNode_ChildrenGroupz1prefix children line_offset last_line_offset_leafNc             C   s   || _ g | _|| _g | _d S )N)	tree_node_children_groupsr!   _node_children)rW   r   r!   r   r   r   rX     s    z_NodesTreeNode.__init__c       	   	   C   s   g }xh| j D ]^\}}}}t|d  }||j |_|dkrbyt||| W n tk
r`   Y nX ||7 }qW || j_x|D ]}| j|_q|W x| j	D ]}|
  qW d S )Nr   )r   r   Zget_first_leafr   rQ   rM   r   r   r!   r   finish)	rW   r   r   Zchildren_partrR   last_line_offset_leafZ
first_leafr#   Z
node_childr   r   r   r     s"    
z_NodesTreeNode.finishc             C   s   | j | d S )N)r   r|   )rW   Z
child_noder   r   r   add_child_node  s    z_NodesTreeNode.add_child_noder   c             C   s4   |d kr|d   }| ||||}| j| d S )Nr   )r0   r   r   r|   )rW   r   r   rR   r   groupr   r   r   add_tree_nodes  s    z_NodesTreeNode.add_tree_nodesc             C   s   d}| j r@| j d }t|j}|jd |j }t||r@|d8 }|tt|d 7 }|rt|dst|dst|d7 }| j	rt
|| j	d |S |S )Nr   r   r   r   r   )r   r   r   r   rR   r1   r    r   r7   r   maxget_last_line)rW   r8   r$   Zchildren_groupr2   r   r   r   r     s    

z_NodesTreeNode.get_last_line)N)r   N)
rN   rO   rP   r   r   rX   r   r   r   r   r   r   r   r   r     s   


r   c               @   sZ   e Zd Zdd Zedd Zdd Zdd Zd	d
 Zdd Z	dd Z
dddZdd ZdS )r[   c             C   s*   t || _| jg| _|| _d| _d| _d S )Nr&   )r   
_base_node_working_stackrV   _prefix_remainderr   )rW   r,   r   r   r   rX     s
    

z_NodesTree.__init__c             C   s   | j d | jS )Nr   )r   r   r   )rW   r   r   r   rm     s    z_NodesTree.parsed_until_linec             C   sd   |j d }xT| jd }|j}|jdkrD|jd j d }||krR|S n|jdkrR|S | j  qW d S )Nr   r   rB   rs   )r   r   r   r   r   r}   )rW   Zindentation_nodeZindentationr#   r   Znode_indentationr   r   r   _get_insertion_node  s    



z_NodesTree._get_insertion_nodec             C   sr   | j }| |}|s$|| j  | _ d S |d jdks6t| |d }|jjdksTt||| | |d  d S )Nr   r5   )rB   rs   r   )r   _remove_endmarkerr   r   r   r   r   _update_tos)rW   
tree_nodesZ
old_prefixr#   r   r   r   rw     s    
z_NodesTree.add_parsed_nodesc             C   sn   |j dkrRt|}|dt|j | jd | | j| | |jd  nt	|rj| |jd  d S )N)rB   rs   r&   r   )
r   r   r   listr   r   r   r|   r   rC   )rW   r   Znew_tosr   r   r   r      s    
z_NodesTree._update_tosc             C   s   |d   }|jdk}d| _|rlt|jd|jd}|dkrl|jd|d  |j|d d  |_| _d| _|r|j| _|dd }|S )zE
        Helps cleaning up the tree nodes that get inserted.
        r   	endmarkerr&   r   r   Nr   )r0   r   r   r   r   rfind)rW   r   r2   Zis_endmarkerZ
separationr   r   r   r   ,  s    
*z_NodesTree._remove_endmarkerc             C   sH   |d j dkrg S | |d  | t| j|||| j\}| _| _|S )zy
        Copies tree nodes from the old parser tree.

        Returns the number of tree nodes that were copied.
        r   )r   
error_node)r   r   _copy_nodesr   r   r   )rW   r   r_   rR   	new_nodesr   r   r   rp   D  s    z_NodesTree.copy_nodesr&   c             C   s@  g }d}xh|D ]`}|j d |kr"P |jdkr.P |jdkrD|jdkrDP t||krdt|rb|| P || qW |sg ||fS |d }	|d }
d}t|
r|
}x|jdkr|jd }qW t|}| ||g |j||\}}}t	|d	k r|
  d}n|st|	| |}d
}|r||d }
|
jdks@t|d r|d}|
  x.|rz|d }
|
 jdkrnP |
  qNW |r6t|d  s|s|d  j}t|d
dd }|r|d }|jdkr|jd }|jdkr|jd }|jd  }|dkstn|d  }|	|||| |}d| _|||fS )Nr&   r   r   r   )r
   r	   r   FrB      T)r   r   r5   )r%   r?   )r@   rA   :)r   r   r   r3   rC   r|   r   r   r   r    r}   r   r   rL   r0   r1   rt   r   r   r   r   )rW   Zworking_stackrE   r_   rR   r   r   Z
new_prefixr#   ZtosZ	last_nodeZhad_valid_suite_lastrB   Z	suite_tosZsuite_nodesZnew_working_stackpZlastr   r   r   r   r   Z  sz    








z_NodesTree._copy_nodesc             C   s   | j   y| j }W n tk
r4   ddg}Y nX t|}t|j}t| j	}t
|dksbtt
|dkr|d  t
|d 7  < n(|d  t
|d 7  < t
|d |d< tdt|| j	| j }| j|_| jj| d S )Nr   r   r   r&   )r   r   rV   r0   
IndexErrorr4   r   r   r   r   r    r   r   tupler   r!   r   r|   )rW   r2   r   r   r   r   r   r   rg     s     


z_NodesTree.closeN)r&   )rN   rO   rP   rX   propertyrm   r   rw   r   r   rp   r   rg   r   r   r   r   r[     s   
_r[   )r&   )'r   r   r(   collectionsr   ZloggingZparso.utilsr   Zparso.python.parserr   Zparso.python.treer   Zparso.python.tokenizer   Zparso.python.tokenr   Z	getLoggerrN   rb   rh   r   r   r   r"   r/   r3   r4   r1   r>   rC   rF   rL   rj   rM   rQ   objectrS   r   r[   r   r   r   r   <module>   s:   
*

 yB