B
    F.\26                 @   s  d Z ddlmZmZ ddlZddlZddlZddlZddlZddl	Z	ddl
Z
dd Zdd Zi Zdd	 ZG d
d deZedG dd deZedG dd deZedG dd deZG dd deddZedG dd deZedG dd deZG dd deZdS ) z*
A pytest plugin to trace resource leaks.
    )print_functiondivisionNc             C   s~   |  d}|jdddddjdtdd	 tD d
d |jddddddd |jdddddd |jddtdddd d S )Nzresource leaksz-Lz--leaksZstoreleakszList of resources to monitor for leaks before and after each test.
Can be 'all' or a comma-separated list of resource names
(possible values: {known_checkers}).
z, c             s   s   | ]}d | V  qdS )z'%s'N ).0sr   r   ?lib/python3.7/site-packages/distributed/pytest_resourceleaks.py	<genexpr>   s    z#pytest_addoption.<locals>.<genexpr>)Zknown_checkers)actiondesthelpz--leaks-timeoutfloatleaks_timeoutg      ?zSWait at most this number of seconds to mark a test leaking
(default: %(default)s).
)r
   typer   defaultr   z--leaks-fail
store_trueleaks_mark_failedFzMark leaked tests failed.)r
   r   r   r   z--leak-retriesleak_retries   zaMax number of times to retry a test when it leaks, to ignore
warmup-related issues (default: 1).
)ZgetgroupZ	addoptionformatjoinsortedall_checkersint)parsergroupr   r   r   pytest_addoption   s4    
$r   c             C   s   |  d}|r|dkr tt}n
|d}tt|tt }|rPtd|f dd |D }t||  d|  d|  d	d
}| j|d d S )Nr   all,zunknown resources: %rc             S   s   g | ]}t |  qS r   )r   )r   Zleakr   r   r   
<listcomp>G   s    z$pytest_configure.<locals>.<listcomp>r   r   r   )checkersgrace_delaymark_failedmax_retriesZleaks_checker)	getvaluer   r   splitset
ValueErrorLeakCheckerZpluginmanagerregister)Zconfigr   unknownr    checkerr   r   r   pytest_configure<   s    


r,   c                s    fdd}|S )Nc                s*   t | tst|  tkst| t < | S )N)
issubclassResourceCheckerAssertionErrorr   )cls)namer   r   decorateT   s    z"register_checker.<locals>.decorater   )r1   r2   r   )r1   r   register_checkerS   s    r3   c               @   s<   e Zd Zdd Zdd Zdd Zdd Zd	d
 Zdd ZdS )r.   c             C   s   d S )Nr   )selfr   r   r   on_start_test_   s    zResourceChecker.on_start_testc             C   s   d S )Nr   )r4   r   r   r   on_stop_testb   s    zResourceChecker.on_stop_testc             C   s   d S )Nr   )r4   r   r   r   on_retrye   s    zResourceChecker.on_retryc             C   s   t d S )N)NotImplementedError)r4   r   r   r   measureh   s    zResourceChecker.measurec             C   s   t d S )N)r8   )r4   beforeafterr   r   r   has_leakk   s    zResourceChecker.has_leakc             C   s   t d S )N)r8   )r4   r:   r;   r   r   r   r   n   s    zResourceChecker.formatN)	__name__
__module____qualname__r5   r6   r7   r9   r<   r   r   r   r   r   r.   ]   s   r.   Zfdsc               @   s$   e Zd Zdd Zdd Zdd ZdS )	FDCheckerc             C   s&   t jdkrdd l}|  S dS d S )Nposixr   )osr1   psutilProcessZnum_fds)r4   rC   r   r   r   r9   u   s    
zFDChecker.measurec             C   s   ||kS )Nr   )r4   r:   r;   r   r   r   r<   |   s    zFDChecker.has_leakc             C   s   d||  S )Nzleaked %d file descriptor(s)r   )r4   r:   r;   r   r   r   r      s    zFDChecker.formatN)r=   r>   r?   r9   r<   r   r   r   r   r   r@   r   s   r@   Zmemoryc               @   s$   e Zd Zdd Zdd Zdd ZdS )RSSMemoryCheckerc             C   s   dd l }|  jS )Nr   )rC   rD   Zmemory_infoZrss)r4   rC   r   r   r   r9      s    zRSSMemoryChecker.measurec             C   s   ||d kS )Ng    cAr   )r4   r:   r;   r   r   r   r<      s    zRSSMemoryChecker.has_leakc             C   s   d|| d  S )Nzleaked %d MB of RSS memoryg    .Ar   )r4   r:   r;   r   r   r   r      s    zRSSMemoryChecker.formatN)r=   r>   r?   r9   r<   r   r   r   r   r   rE      s   rE   Zthreadsc               @   s$   e Zd Zdd Zdd Zdd ZdS )ActiveThreadsCheckerc             C   s   t t S )N)r&   	threading	enumerate)r4   r   r   r   r9      s    zActiveThreadsChecker.measurec             C   s
   ||k S )Nr   )r4   r:   r;   r   r   r   r<      s    zActiveThreadsChecker.has_leakc             C   s(   || }|st dt|t|tdf S )Nzleaked %d Python threads: %s)key)r/   lenr   str)r4   r:   r;   leakedr   r   r   r      s    zActiveThreadsChecker.formatN)r=   r>   r?   r9   r<   r   r   r   r   r   rF      s   rF   c               @   s   e Zd Zedd ZdS )_ChildProcessc             C   s   | |j | | S )N)pidr1   cmdline)r0   pr   r   r   from_process   s    z_ChildProcess.from_processN)r=   r>   r?   classmethodrQ   r   r   r   r   rM      s   rM   )rN   r1   rO   Z	processesc               @   s$   e Zd Zdd Zdd Zdd ZdS )ChildProcessesCheckerc          
   C   s   dd l }i }| }x|jddD ]}y| p | |jkrtj|	 t
jr| }tdd |D rpw"tdd |D rw"t|||j| f< W d Q R X W q" |jk
r   Y q"X q"W |S )Nr   T)	recursivec             s   s   | ]}| d V  qdS )z2from multiprocessing.semaphore_tracker import mainN)
startswith)r   ar   r   r   r	      s   z0ChildProcessesChecker.measure.<locals>.<genexpr>c             s   s   | ]}| d V  qdS )z+from multiprocessing.forkserver import mainN)rU   )r   rV   r   r   r   r	      s   )rC   rD   childrenZoneshotZppidrN   rB   pathsamefileZexesys
executablerO   anyrM   rQ   Zcreate_timeZNoSuchProcess)r4   rC   rW   rP   crO   r   r   r   r9      s$    
"

&
zChildProcessesChecker.measurec             C   s   t |t |k S )N)r&   )r4   r:   r;   r   r   r   r<      s    zChildProcessesChecker.has_leakc             C   s^   t |t | }|stg }x*t|D ]}|| }|dj|d q&W dt|d|f S )Nz7  - pid={p.pid}, name={p.name!r}, cmdline={p.cmdline!r})rP   zleaked %d processes:
%s
)r&   r/   r   appendr   rJ   r   )r4   r:   r;   rL   Z	formattedrI   rP   r   r   r   r      s    zChildProcessesChecker.formatN)r=   r>   r?   r9   r<   r   r   r   r   r   rS      s   rS   tracemallocc               @   s<   e Zd Zdd Zdd Zdd Zdd Zd	d
 Zdd ZdS )TracemallocMemoryCheckerc             C   s   dd l a d S )Nr   )r`   )r4   r   r   r   __init__   s    z!TracemallocMemoryChecker.__init__c             C   s   t d d S )Nr   )r`   start)r4   r   r   r   r5      s    z&TracemallocMemoryChecker.on_start_testc             C   s   t   d S )N)r`   stop)r4   r   r   r   r6      s    z%TracemallocMemoryChecker.on_stop_testc             C   s$   dd l }| \}}| }||fS )Nr   )r`   Zget_traced_memoryZtake_snapshot)r4   r`   ZcurrentZpeakZsnapr   r   r   r9      s    z TracemallocMemoryChecker.measurec             C   s   |d |d d kS )Nr   g    .Ar   )r4   r:   r;   r   r   r   r<      s    z!TracemallocMemoryChecker.has_leakc             C   s   |\}}|\}}| |d}d}d}	g }
|
d|| d  g7 }
xd|d | D ]T}|jpZ|j}||	k rfP |jpp|j}|
d|d |f g7 }
|
dd |j D 7 }
qLW d	|
S )
N	traceback   g     jAz&leaked %.1f MB of traced Python memoryg    .Az"  - leaked %.1f MB in %d calls at:c             S   s   g | ]}d | qS )z    r   )r   liner   r   r   r      s    z3TracemallocMemoryChecker.format.<locals>.<listcomp>r^   )Z
compare_to	size_diffsizeZ
count_diffcountre   r   r   )r4   r:   r;   Zbytes_beforeZsnap_beforeZbytes_afterZ
snap_afterZdiffZndiffZmin_size_difflinesstatrh   rj   r   r   r   r      s     zTracemallocMemoryChecker.formatN)	r=   r>   r?   rb   r5   r6   r9   r<   r   r   r   r   r   ra      s   ra   c               @   s   e Zd Zdd Zdd Zdd Zdd Zd	d
 Zdd ZdddZ	e
jdddd Ze
jdddd Ze
jddddd Ze
jddddd Ze
jdd ZdS )r(   c             C   s@   || _ || _|| _|| _i | _i | _i | _tt	| _
d| _d S )NF)r    r!   r"   r#   skip_checkerscountersr   collectionsdefaultdictr&   outcomes	_retrying)r4   r    r!   r"   r#   r   r   r   rb     s    zLeakChecker.__init__c             C   s   t   d S )N)gcZcollect)r4   r   r   r   cleanup  s    zLeakChecker.cleanupc                s    fddj D S )Nc                s"   g | ]}|j  d kr|qS )r   )rm   get)r   r]   )nodeidr4   r   r   r     s    z/LeakChecker.checks_for_item.<locals>.<listcomp>)r    )r4   rv   r   )rv   r4   r   checks_for_item  s    zLeakChecker.checks_for_itemc             C   s   dd |  |D S )Nc             S   s   g | ]}||  fqS r   )r9   )r   r]   r   r   r   r     s    z'LeakChecker.measure.<locals>.<listcomp>)rw   )r4   rv   r   r   r   r9     s    zLeakChecker.measurec             C   s\   x|  |D ]}|  qW x:| |D ],\}}|d k	s<t| j| | |d f q(W d S )N)rw   r5   r9   r/   rn   r_   )r4   rv   r+   r:   r   r   r   measure_before_test  s
    zLeakChecker.measure_before_testc       
         s   j   }|st|dhkr d S  fdd}t }|j }| }|rt  x|D ]\}}}|  qXW | }xH|rt |k rtd   x|D ]\}}}|  qW | }qvW |r|j < nj d  x	 D ]}	|	
  qW d S )NZpassedc                 sn   g } xd  D ]V\}}|d k	s$tj  | }|d \}}||f|d< |||r| |||f qW | S )N)r9   r/   rn   r<   r_   )r   r+   r;   r]   r:   _)rv   r4   r   r   run_measurements,  s    z8LeakChecker.measure_after_test.<locals>.run_measurementsg?)rq   r/   timer!   rt   r7   Zsleepr   poprw   r6   )
r4   rv   rq   r{   Zt1Zdeadliner   r]   rz   r+   r   )rv   r4   r   measure_after_test%  s0    




zLeakChecker.measure_after_testNc       	   
      s    fdd} j }| j|}|rd| _zjyxt| jD ]
}|  q:W W n: tk
r } ztd dd l}|	  W d d }~X Y nX | j|}W d d| _X |S )Nc                 s&   ddl m}     |  dd}d S )Nr   )runtestprotocolF)nextitemlog)Z_pytest.runnerr   Z_initrequest)r   Zreports)itemr   r   r   run_test_againP  s    z/LeakChecker.maybe_retry.<locals>.run_test_againTz&--- Exception when re-running test ---r   F)
rv   r   ru   rr   ranger#   	Exceptionprintre   	print_exc)	r4   r   r   r   rv   r   iere   r   )r   r   r   maybe_retryO  s    zLeakChecker.maybe_retryT)hookwrapperc             #   s   | j s|j}|| jkstdd | jD | j|< |d}|d k	rtt|jtt	 }|rjt
d|f tdd |jD   fdd| jD | j|< d V  d S )	Nc             S   s   i | ]
}g |qS r   r   )r   r]   r   r   r   
<dictcomp>y  s    z7LeakChecker.pytest_runtest_protocol.<locals>.<dictcomp>leakingz)pytest.mark.leaking: unknown resources %rc             s   s   | ]}t | V  qd S )N)r   )r   rV   r   r   r   r	     s    z6LeakChecker.pytest_runtest_protocol.<locals>.<genexpr>c                s   h | ]}t | r|qS r   )
isinstance)r   r]   )classesr   r   	<setcomp>  s    z6LeakChecker.pytest_runtest_protocol.<locals>.<setcomp>)rr   rv   rn   r/   r    Z
get_markerr   r&   argsr   r'   tuplerm   )r4   r   r   rv   r   r*   r   )r   r   pytest_runtest_protocolt  s    

z#LeakChecker.pytest_runtest_protocolc             c   s   |  |j d V  d S )N)rx   rv   )r4   r   r   r   r   pytest_runtest_setup  s    z LeakChecker.pytest_runtest_setup)r   Ztrylastc             c   s8   d V  |  |j | js4| |}|r4| jr4t  d S )N)r~   rv   rr   r   r"   pytestZfail)r4   r   r   r   r   r   pytest_runtest_teardown  s    

z#LeakChecker.pytest_runtest_teardownc             #   s   |j  | j  }||j d V }| js|jdkr| j|j }|r| jrv|	d d|_d
 fdd|D |_n
|	d d S )NZteardown)failedLLEAKEDr   r^   c                s&   g | ]\}}}d  | ||f qS )z%s %s)r   )r   r+   r:   r;   )rv   r   r   r     s   z8LeakChecker.pytest_report_teststatus.<locals>.<listcomp>)rL   r   r   )rv   rq   addoutcomerr   Zwhenr   ru   r"   Zforce_resultr   Zlongrepr)r4   Zreportrq   r   r   r   )rv   r   pytest_report_teststatus  s    



z$LeakChecker.pytest_report_teststatusc       
   
   C   sl   |}| d}|rh|dd xH|D ]@}|j}x4| j| D ]&\}}}	|d|j|||	f  q:W q$W d S )NrL   =zRESOURCE LEAKSz%s %s)Z
getreportsZ	write_seprv   r   rg   r   )
r4   ZterminalreporterZ
exitstatusZtrrL   Zreprv   r+   r:   r;   r   r   r   pytest_terminal_summary  s    

z#LeakChecker.pytest_terminal_summary)N)r=   r>   r?   rb   rt   rw   r9   rx   r~   r   r   Zhookimplr   r   r   r   r   r   r   r   r   r(      s   *
%
r(   )__doc__Z
__future__r   r   ro   rs   r|   rB   rZ   rG   r   r   r,   r   r3   objectr.   r@   rE   rF   
namedtuplerM   rS   ra   r(   r   r   r   r   <module>   s4   +
+*