B
    4\                 @   sH  d dl mZmZmZ d dlZd dlmZ d dlmZ d dl	Z	d dl
mZmZ d dlZd dlmZ d dlZd dlmZ d dlmZmZmZmZmZ d d	lmZ d dlZd d
lmZmZmZmZm Z  d dl!m"Z"m#Z# d dl$m%Z%m&Z& d dl'mZ d dl(mZ d dl)m*Z* d dl+m,Z,m-Z- d dl.m/Z/ d dl0m1Z1m2Z2m3Z3m4Z4m5Z5m6Z6m7Z7m8Z8m9Z9m:Z:m;Z; d dl0m<Z<m=Z= d dl>m?Z? dZ@dZAedd ZBe3 dd ZCe3ddgddd ZDe3dd d!d" ZEe3dd d#d$ ZFe3ddgd% dd&d' ZGe3ddgd% dd(d) ZHe3ddgd% dd*d+ ZIe3ddgd% dd,d- ZJe3ddgd% dd.d/ ZKe3dg dd0d1 ZLe3g d2d3d4 ZMe3 d5d6 ZNe3 d7d8 ZOe3 d9d: ZPe3d;d<id=d>d? ZQe3d;d<idd@dAdB ZRe3 dCdD ZSe3 dEdF ZTe3 dGdH ZUe3 dIdJ ZVe3dd dKdL ZWe3ddgddMdN ZXe3 dOdP ZYdQdR ZZdSdT Z[e3 dUdV Z\e3dedWdXdY Z]e3 dZd[ Z^e3ed\d]d^ Z_e4 d_d` Z`e4 dadb Zaejbjcejdedcdddee3g d2dfdg Zfe3 dhdi Zge3g d2djdk Zhdldm Zie3dd dndo Zje3g ddpdqdr Zke3g d2dsdt Zle3ddudvdwdx Zme3dd dydz Zne3ddgd{ dd|d} Zoe3ddgd{ dd~d Zpe3dd dd Zqe;ejbjcejdedcdddeejbjcejrdk ddee3dg dddd Zse;e=e3dd dd Zte;e=e3dd dd Zue3dd dd Zve=e3ddgd ddd Zwe=e3ddgd ddd Zxe3dd dd Zye3dd dd Zze3dd dd Z{e3ddgddd Z|e3dg ddd Z}e3dd dd Z~dd Ze3dd dd Ze3dd dd Ze;e3dedddd Ze3dddvdd Ze3ddeddd Ze3ddgddd Ze4 dd Zejbjddee3dg ddd Ze3ddgd% ddd Zejbjcejded ddee4 dd Ze3dddiddd Ze3dddiddd Ze3dd ddĄ Ze3dedWddƄ Ze3ddgd dddɄ Ze3dd dd˄ Zdd̈́ Ze3dd ddτ Zejbjddee3ddgd% ddddӄ Ze3dd ddՄ Ze3dd ddׄ Ze3dd ddل Ze3dd ddۄ Ze3dd dd݄ Ze3dd dd߄ Ze3 dd Ze3ddgdddiiddd Ze3dd dd ZdS )    )print_functiondivisionabsolute_importN)defaultdict)	timedelta)addmul)sleep)delayed)mergeconcatvalmapfirstfrequencies)gen)NannyWorkerClientwaitfire_and_forget)connectrpc)	Scheduler	BANDWIDTH)r   )time)dumps)dumps_function
dumps_task)tmpfile)incdecgen_clustergen_testslowincslowaddslowdecclusterdivvaryingslow)loopnodebug)applyz
alice:1234zbob:1234c               C   s   dS )Nr    r-   r-   r-   ?lib/python3.7/site-packages/distributed/tests/test_scheduler.py<lambda>$   s    r/   c             C   s^   t | jtst| jt| ks"ttt| j t| ks@ttt| jt| ksZtd S )N)	
isinstanceaddressstrAssertionErrorsumncoresvaluesreprlen)sabr-   r-   r.   test_administration'   s    r<   T)z	127.0.0.1   )clientr5   c             c   s   t td}t t|}| |}t|gV  |j|j j|j|j hksNt	t t
||}| |}x:|j|jks|j|j s|j|j jst	tdV  qhW d S )Nr=   g-C6?)r
   r   persistr   taskskeywho_hasworkersr1   r3   r   r   r	   )cr9   r:   xyfzf2r-   r-   r.   test_respect_data_in_memory/   s    
 
rJ   )r>   c       
      c   s   t td}t t|}| |}t|V  x6|j|j jsT|j|jksT|j|jkrbt	dV  q.W t t
|}| |}|V }	|	dkstd S )Nr=   g{Gz?)r
   r   r?   r   r@   rA   rB   datar   r	   r    computer3   )
rD   r9   r:   r;   rE   rF   yyrH   zzresultr-   r-   r.   test_recompute_released_results?   s    

(
rP   c             #   s   | j ttddd jd| j ttdddjdgV }ttt| }dd |D }| |}t|V  t fdd	|d d d D tfd
d	|dd d D  }|dkst	d S )Nr   d      )rC   r=   c             S   s   g | ]}t t|qS r-   )r
   r   ).0rE   r-   r-   r.   
<listcomp>U   s    zCtest_decide_worker_with_many_independent_leaves.<locals>.<listcomp>c             3   s   | ]}|j  jkV  qd S )N)rA   rK   )rS   rF   )r:   r-   r.   	<genexpr>Z   s    zBtest_decide_worker_with_many_independent_leaves.<locals>.<genexpr>c             3   s   | ]}|j  jkV  qd S )N)rA   rK   )rS   rF   )r;   r-   r.   rU   [   s    P   )
scatterlistranger1   r   zipr?   r   r4   r3   )rD   r9   r:   r;   ZxsZysZy2sZnhitsr-   )r:   r;   r.   /test_decide_worker_with_many_independent_leavesP   s     

"r[      c             c   sB   | j td|j|jgd}t|V  |j|jks>|j|jks>td S )Nr=   )rC   )submitr   r1   r   rA   rK   r3   )r>   r9   r:   r;   rD   rE   r-   r-   r.   $test_decide_worker_with_restrictions`   s    
r^   c             c   sX   | j dg|jdV \}| jt||j|jgd}t|V  |j|jksT|j|jksTtd S )Nr=   )rC   )rW   r1   r]   r   r   rA   rK   r3   )r>   r9   r:   r;   rD   rE   rF   r-   r-   r.   &test_move_data_over_break_restrictionsg   s    
r_   c             c   sz   | j dddgg|jd| j dg|jdgV \\}\}| jtd|j|jgd}t|V  |j|j j|j|j hksvt	d S )Nr=   rR   r\   )rC   )
rW   r1   r]   r   r   r@   rA   rB   rC   r3   )r>   r9   r:   r;   rD   rE   rF   rH   r-   r-   r.   test_balance_with_restrictionso   s
    
r`   c          	   c   sn   | j tddd}x|js&tdV  qW |j|j |jks>tt	tj
 ttdd|V  W d Q R X d S )Nr=   z127.0.0.5:9999)rC   g{Gz?2   )milliseconds)r]   r   r@   r   r	   rA   
unrunnabler3   pytestraisesTimeoutErrorwith_timeoutr   )r>   r9   r:   r;   rD   rE   r-   r-   r.   test_no_valid_workersy   s    rh   c             c   s(   | j tdddd}|V }|dks$td S )Nr=   z127.0.0.5:9999T)rC   allow_other_workersrR   )r]   r   r3   )r>   r9   r:   r;   rD   rE   rO   r-   r-   r.   (test_no_valid_workers_loose_restrictions   s    
rj   c          	   c   sj   |  td}x|js"tdV  qW |j|j |jks:tt	tj
 ttdd|V  W d Q R X d S )Nr=   g{Gz?ra   )rb   )r]   r   r@   r   r	   rA   rc   r3   rd   re   rf   rg   r   )r>   r9   rE   r-   r-   r.   test_no_workers   s    rk   )r5   c             c   s   | j g dV  d S )N)rC   )retire_workers)r9   r-   r-   r.   test_retire_workers_empty   s    rm   c             C   sj   | j ttdfttdfdg dgddgdd | js<t| jsFt| jdd | jr\t| jrftd S )Nr=   rE   )rE   rF   rF   ident)r@   dependencieskeysr>   )r>   )update_graphr   r   r@   r3   ro   remove_client)r9   r:   r;   r-   r-   r.   test_remove_client   s    



rs   c          	   c   sJ   t | j6}| V }|d dks&t|d  ds<tW d Q R X d S )Ntyper   id	scheduler)r   r1   identityr3   lower
startswith)r9   r:   r;   rrn   r-   r-   r.    test_server_listens_to_other_ops   s    
r{   c             C   s   dd t dD }| jtt|t|dd |D d |j| jksFt| j|jd |j| j	ksdtt
| j|j jt
|kst|   d S )Nc             S   s   i | ]}t |fd | qS )zx-%d)r   )rS   ir-   r-   r.   
<dictcomp>   s    z5test_remove_worker_from_scheduler.<locals>.<dictcomp>   c             S   s   i | ]}t  |qS r-   )set)rS   kr-   r-   r.   r}      s    )r@   rp   ro   )r1   )rY   rq   r   r   rX   r1   stream_commsr3   remove_workerr5   r8   rC   
processingvalidate_state)r9   r:   r;   dskr-   r-   r.   !test_remove_worker_from_scheduler   s    r   z*distributed.scheduler.events-cleanup-delayz10 ms)configc             c   s   |j | jkst|j | jks t|j | jks0t|j | jks@t| j|j d |j | jks^t|j | jksnt|   t }x.|j | jkrtdV  t |d k s~tq~W |j | jkstd S )N)r1   g{Gz?rR   )	r1   eventsr3   r5   r   r   r   r   r	   )r9   r:   r;   startr-   r-   r.    test_clear_events_worker_removal   s    r   )r   r>   c             c   s   | j |jkst|| j  | j |jks,t| j |jks<t| |jksJt|| j  t }x.| j |jkrtdV  t |d k s^tq^W d S )Ng{Gz?rR   )ru   r   r3   rr   Zclientsr   r   r	   )rD   r9   r:   r;   r   r-   r-   r.    test_clear_events_client_removal   s    r   c             c   s   t | j| jdd}d|jd< d|jd< |dV  dd	 td
D }| jtt|t	|ddd	 |D d | j
|jt	|j|j| jd |   |j| jkst| j|j d |j|j|jhkst| V  d S )Nr\   )r5      zx-5r=   rF   r   c             S   s   i | ]}t |fd | qS )zx-%d)r   )rS   r|   r-   r-   r.   r}      s    z#test_add_worker.<locals>.<dictcomp>
   r>   c             S   s   i | ]}t  |qS r-   )r   )rS   r   r-   r-   r.   r}      s    )r@   rp   r>   ro   )r1   rp   r5   servicesZ	addresses)r   ipportrK   _startrY   rq   r   r   rX   
add_workerr1   r5   r   r   	host_infor3   _close)r9   r:   r;   wr   r-   r-   r.   test_add_worker   s    

$r   c             c   sr   dd }t | jV }|dt|ddV  x6tdD ]*}| V }t| j}t	||ks6t
q6W | V  d S )Nc             S   s   t t| jS )N)r   dictworker_info)rv   r-   r-   r.   func  s    ztest_feed.<locals>.funcfeedg{Gz?)opfunctioninterval   )r   r1   writer   rY   readr   r   cloudpickleloadsr3   close)r9   r:   r;   r   commr|   responseexpectedr-   r-   r.   	test_feed   s    

r   c       
      c   s   dd }dd }dd }t | jV }|dt|t|t|dd	V  x&td
D ]}| V }|dksRtqRW | V  t }	x,t	| dst
dV  t |	 d
k stqW d S )Nc             S   s   dS )Nr=   r-   )rv   r-   r-   r.   setup  s    z'test_feed_setup_teardown.<locals>.setupc             S   s   |dkst dS )Nr=   OK)r3   )rv   stater-   r-   r.   r     s    z&test_feed_setup_teardown.<locals>.funcc             S   s
   d| _ d S )NZdone)flag)rv   r   r-   r-   r.   teardown  s    z*test_feed_setup_teardown.<locals>.teardownr   g{Gz?)r   r   r   r   r   r   r   r   )r   r1   r   r   rY   r   r3   r   r   hasattrr   r	   )
r9   r:   r;   r   r   r   r   r|   r   r   r-   r-   r.   test_feed_setup_teardown  s"    

r   c             #   sz   t d}|d  fdd}t| jV }|dt|ddV  x&tdD ]}| V }|d	ksNt	qNW |
 V  d S )
Nnumpyi c                s    }dS )NTr-   )rv   rF   )rE   r-   r.   r   5  s    z(test_feed_large_bytestring.<locals>.funcr   g?)r   r   r   r   T)rd   importorskipZonesr   r1   r   r   rY   r   r3   r   )r9   r:   r;   npr   r   r|   r   r-   )rE   r.   test_feed_large_bytestring/  s    


r   c             c   s   |  ddddV }dd |j D ddd	hks6tt|jt|jB ddd	hksXtt|j|jddddksvt|d= |d= t }x<t|jt|jB d	hkrt	d
V  t |d k stqW d S )Nr=   rR   r\   )rE   rF   rH   c             S   s   h | ]}|j r|jqS r-   )rB   rA   )rS   tsr-   r-   r.   	<setcomp>I  s    z#test_delete_data.<locals>.<setcomp>rE   rF   rH   g{Gz?r   )
rW   r@   r6   r3   r   rK   r   r   r   r	   )rD   r9   r:   r;   dr   r-   r-   r.   test_delete_dataE  s    ""r   c             c   sh   |  td}|V  |j|jks"t| |V  t }x.|j|jkrbtdV  t |d k s6tq6W d S )Nr=   g{Gz?r   )	r]   r   rA   rK   r3   Z_cancelr   r   r	   )rD   r9   r:   rE   r   r-   r-   r.   test_deleteV  s    r   c             c   s:  t | jV }t | jV }|dddV  |dddV  | V  | V  t| jddhksft|dttdfttdfdg dgddd	gd
V  |dttdftt	ddfdg dgdddgd
V  | V \}|d dkst|d d	kst| V \}|d dks$t|d dks6td S )Nzregister-clientrD   )r   r>   rG   zupdate-graphr=   rE   )rE   rF   rF   )r   r@   ro   r>   rp   r   )rE   rH   rH   r   zkey-in-memoryrA   )
r   r1   r   r   r   Zclient_commsr3   r   r   r   )r9   r:   r;   rD   rG   msgr-   r-   r.   test_filtered_communicationd  s2    





r   c              C   sJ   t t} t| ddkstt t}| |ks2tt t}| |ksFtd S )Nr      )r   r   r   r   r3   r    )r:   r;   rD   r-   r-   r.   test_dumps_function  s    r   c              C   s   t tdf} t| ddhks tddd}t t|ddd	if} t| d ddd
ksZtt| d dksptt| d dd	ikstt t|df} t| d ddd
kstt| d dkstt| ddhkstd S )Nr=   r   argsrR   c             S   s   | | S )Nr-   )rE   rF   r-   r-   r.   r/     s    z!test_dumps_task.<locals>.<lambda>)r=   rF   r   r\   kwargs)rR   )r   r   r   r3   r,   r   r   )r   rG   r-   r-   r.   test_dumps_task  s    
r   c             C   s   | j dd tdD dd tdD ddd tdD d td	d
 | j D sXt| j|jd t| j|jhks|ttdd
 | j D std S )Nc             S   s   i | ]}t t|fd | qS )zx-%d)r   r   )rS   r|   r-   r-   r.   r}     s    z,test_ready_remove_worker.<locals>.<dictcomp>r~   c             S   s   g | ]}d | qS )zx-%dr-   )rS   r|   r-   r-   r.   rT     s    z,test_ready_remove_worker.<locals>.<listcomp>r>   c             S   s   i | ]}g d | qS )zx-%dr-   )rS   r|   r-   r-   r.   r}     s    )r@   rp   r>   ro   c             s   s   | ]}t |j|jkV  qd S )N)r8   r   r5   )rS   r   r-   r-   r.   rU     s    z+test_ready_remove_worker.<locals>.<genexpr>)r1   c             s   s   | ]}t |j|jkV  qd S )N)r8   r   r5   )rS   r   r-   r-   r.   rU     s    )	rq   rY   allrC   r6   r3   r   r1   r   )r9   r:   r;   r-   r-   r.   test_ready_remove_worker  s    r   )r>   r   c             c   sv   |  ttd}t|V  | V  t|jdks6tx&|j D ]}|j	rPt|j
rBtqBW |jrht|jrrtd S )Nr~   rR   )mapr   rY   r   Zrestartr8   rC   r3   r6   	occupancyr   r@   ro   )rD   r9   r:   r;   futureswsr-   r-   r.   test_restart  s    



r   c             c   s   | j ddidV }||jd|jdiks*t| j ddi|jgdV }||jdiksTt| j ddi|jgdV }||jd|jdikstd S )Nr   Zping)r   s   pong)r   rC   )r   hosts)	broadcastr1   r3   r   )r9   r:   r;   rO   r-   r-   r.   test_broadcast  s    r   )r   c             c   s   | j ddiddV }tdd | D s.t| j ddi|jgddV }t|dksXtt| d	 |jksrt| j ddi|jgdd
V }||kstd S )Nr   rw   T)r   nannyc             s   s   | ]}|d  dkV  qdS )rt   r   Nr-   )rS   r   r-   r-   r.   rU     s    z'test_broadcast_nanny.<locals>.<genexpr>)r   rC   r   r=   ru   )r   r   r   )	r   r   r6   r3   worker_addressr8   r   ru   r   )r9   r:   r;   Zresult1Zresult2Zresult3r-   r-   r.   test_broadcast_nanny  s    


r   c           	   c   s   t dd} | d t| j| jdd}| V  | j|j jdksFt	| j
d |jksZt	tt, t| j| jdd}| V  | V  W d Q R X |  V  | V  d S )NT)validater   alice)name)r   r   r   r   r   r   rC   r1   r   r3   aliasesrd   re   
ValueErrorr   r   )r9   r   w2r-   r-   r.   test_worker_name  s    




r   c           	   c   s  t jddi tdd} | d td| j| j t| j	| j
dd}t| j	| j
d	d}td
| j
dd}| | | gV  | ddkst| ddkst| ddkst| ddkst| ddkst| ddkst| |j|jkst| d|jkst| d	|jks4t| d|jksJt| d
d
ks^t| d|j	kstt| d	|j	kst| d|j	kst| ddkst| jddddkst|  V  dd |||gD V  W d Q R X d S )Nz!distributed.comm.timeouts.connect100msT)r   r   z
scheduler:r   )r   {   z	127.0.0.1Zcharliez127.0.0.1:8000ztcp://127.0.0.1:8000z
[::1]:8000ztcp://[::1]:8000zlocalhost:8000)ztcp://127.0.0.1:8000ztcp://[::1]:8000Zjimmyz	zzzt:8000F)Zresolveztcp://zzzt:8000c             S   s   g | ]}|  qS r-   )r   )rS   r   r-   r-   r.   rT     s    z'test_coerce_address.<locals>.<listcomp>)daskr   r   r   r   printr1   Zlisten_addressr   r   r   r   Zcoerce_addressr3   Zcoerce_hostnamer   )r9   r:   r;   rD   r-   r-   r.   test_coerce_address  s4    


r   winz#file descriptors not really a thing)reasonc             c   s   t d}| }| }t| j| j}|dV  | V  | }t	 }x.| |krzt
dV  t	 |d k sNtqNW d S )Npsutilr   g{Gz?r   )rd   r   Processnum_fdsr   r   r   r   r   r   r   r	   r3   )r9   r   procZbeforer   Zduringr   r-   r-   r.   test_file_descriptors_dont_leak  s    

r   c             C   s\   | j ttdfttdfttdfddgdg g ddd d	| jksJtd	| jksXtd S )
Nr=   rE   rR   )rE   rF   rH   rF   )rF   rE   rH   r>   )r@   rp   ro   r>   rH   )rq   r   r   r@   r3   ro   )r9   r:   r;   r-   r-   r.   test_update_graph_culls&  s    

r   c             C   s<   | j tddd t| j}| j tdd | j| jks8td S )Nr=   F)r1   r5   resolve_address)r1   r   )r   r   r   r5   r3   )r9   r5   r-   r-   r.   test_add_worker_is_idempotent2  s    
r   c             C   s   t | dd}|j| kstd S )NT)r*   r   )r   Zio_loopr3   )r*   r9   r-   r-   r.   test_io_loop:  s    r   c             #   s   t tdt t}| |}t|gV   js8t j}t fdd|D s^tt	|t	 jk stttfdd|D stt	 j|jt	|kstd S )Nr=   c             3   s   | ]}| j kV  qd S )N)transition_log)rS   line)r9   r-   r.   rU   I  s    ztest_story.<locals>.<genexpr>c             3   s*   | ]"} j |d  kp  j |d kV  qdS )r   N)rA   )rS   r   )rE   r-   r.   rU   K  s    )
r
   r   r?   r   r   r3   storyrA   r   r8   )rD   r9   r:   r;   rF   rG   r   r-   )r9   rE   r.   
test_story?  s    

r   )r5   r>   c          	   c   s   t tj |jddidddV  W d Q R X t }t tj | jdddV  W d Q R X t |d k sntt|j|j	d	d
}| jddidd|
 gV  |jd dkst| V  d S )NrE   r=   r   g?)rK   r>   timeoutr   )r   g      ?r\   )r5   rF   rR   r   )rK   r   )rd   re   r   rf   rW   r   r3   r   r   r   r   rK   r   )rD   r9   r   r   r-   r-   r.   test_scatter_no_workersP  s     r   c             c   sD   t | j| jddd}|dV  | j|j jdks6t| V  d S )Nr\   i90  )r5   memory_limitr   )	r   r   r   r   rC   r1   r   r3   r   )r9   r   r-   r-   r.   !test_scheduler_sees_memory_limitsb  s    r   i  )r>   r   c             c   s   | j dg|jdV \}| j ttdg|jdV \}| |jgksHt| V }t||jgksft||j d |jks~tt|j|jgkst| g kst|j|j j	|j
|j |j
|j hkst| V }|rtd S )Nr=   )rC   i  r5   )rW   r1   rX   rY   workers_to_closer3   rl   r5   rC   has_whatr@   rA   )rD   r9   r:   r;   rE   rF   rC   r-   r-   r.   test_retire_workersk  s    

r   c             c   s   |j dddV  t|jdks"t|j dddV  t|jdksDt|j dddV  t|jdksft|j dddV  t|jdkstx(|jdr|jdstdV  qW d S )Nr=   T)nclose_workersr   Zclosg{Gz?)rl   r8   rC   r3   statusry   r   r	   )rD   r9   r:   r;   r-   r-   r.   test_retire_workers_n  s    r      c             '   s   d j d< d j d< d j d< | jtdddgdddgd	}x,td
d  j D dk rftdV  q<W   }t	 fdd|D st
t|dkst
d S )Nr   r:   r;   r=   rD   za-4zb-4zc-1)rA   c             s   s   | ]}t |jV  qd S )N)r8   r   )rS   r   r-   r-   r.   rU     s    z(test_workers_to_close.<locals>.<genexpr>r\   gMbP?c             3   s   | ]} j | j V  qd S )N)rC   r   )rS   r   )r9   r-   r.   rU     s    )task_durationr   r#   r4   rC   r6   r   r	   r   r   r3   r8   )Zclr9   rC   r   Zwtcr-   )r9   r.   test_workers_to_close  s    


r   c             '   sP  |d j d|d j d|d j d|d j di  fdd}t|j|d	td
d |D ks^t| jtdd|d j d}xt|jdk rt	dV  qxW t|j|d	|d j |d j hkst~xt|jdkrt	dV  qW | j
d|d j dV }| j
d|d j dV }| j
d|d j dV }t|j|d	|d j |d j hksLtd S )Nr   r:   r=   rR   r;   r\   c                s
    | j  S )N)r1   )r   )groupsr-   r.   rA     s    z*test_workers_to_close_grouped.<locals>.key)rA   c             s   s   | ]}|j V  qd S )N)r1   )rS   r   r-   r-   r.   rU     s    z0test_workers_to_close_grouped.<locals>.<genexpr>g?)delayrC   gMbP?Zdaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa)rC   ZKbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb)r1   r   r   r3   r]   r#   r8   Zrprocessingr   r	   rW   )rD   r9   rC   rA   futureavZbvZbv2r-   )r   r.   test_workers_to_close_grouped  s$    r   c             c   sT   | j tdd|jdd}tdV  |j|jgdV  tdd |j D sPt	d S )	NrQ   g      ?T)r   rC   ri   g?)rC   c             s   s   | ]}|j d kV  qdS )r   N)Z
suspicious)rS   r   r-   r-   r.   rU     s    z:test_retire_workers_no_suspicious_tasks.<locals>.<genexpr>)
r]   r#   r1   r   r	   rl   r   r@   r6   r3   )rD   r9   r:   r;   r   r-   r-   r.   'test_retire_workers_no_suspicious_tasks  s    r   )r\   r   zintermittent failure   )r>   r5   r   c             #   s  t dV  td}td}| }| }d} fddt|D }dd |D V  xt j|k rxt dV  q\W | }t dV  | }	|	|| kst	|j
j
d	d
d}
| |
}
t|
V  | }||d|  kst	| |
|
j }t|V  | }||| k st	t dV  | }||| k s@t	dd |D V   jjr^t	| jjrlt	 jrxt	t }x8| || krt dV  t |d k st	qW d S )Ng?r   z
dask.arrayr~   c                s    g | ]}t  j j jd qS ))r*   )r   r   r   r*   )rS   r|   )r9   r-   r.   rT     s    z)test_file_descriptors.<locals>.<listcomp>c             S   s   g | ]}|  qS r-   )r   )rS   r   r-   r-   r.   rT     s    g?)i  i  )   r   )sizeZchunksrR   r=   c             S   s   g | ]}|  qS r-   )r   )rS   r   r-   r-   r.   rT     s    g{Gz?r\   )r   r	   rd   r   r   r   rY   r8   r5   r3   randomr?   r   Tr   openr   r   )rD   r9   r   Zdar   Z	num_fds_1NnanniesZ	num_fds_2Z	num_fds_3rE   Z	num_fds_4rF   Z	num_fds_5Z	num_fds_6r   r-   )r9   r.   test_file_descriptors  sD    




r  c             c   s   | j ttddd}x,tdd |j D dk r@tdV  qW d|j  k rXdk s^n t	x4||gD ](}d	|j
|j j  k rd
k shn t	qhW d S )Ni  g?)r   c             s   s   | ]}t |jV  qd S )N)r8   rB   )rS   r   r-   r-   r.   rU   	  s    z'test_learn_occupancy.<locals>.<genexpr>r   g{Gz?rQ   ra   i  )r   r#   rY   r4   r@   r6   r   r	   total_occupancyr3   rC   r1   r   )rD   r9   r:   r;   r   r   r-   r-   r.   test_learn_occupancy  s    r  c             c   s^   | j ttddd}x(tdd |j D s<tdV  qW d|j  k rTdk sZn t	d S )Ni  g?)r   c             s   s   | ]}|j V  qd S )N)rB   )rS   r   r-   r-   r.   rU     s    z)test_learn_occupancy_2.<locals>.<genexpr>g{Gz?rQ   )
r   r#   rY   anyr@   r6   r   r	   r  r3   )rD   r9   r:   r;   r   r-   r-   r.   test_learn_occupancy_2  s    r  c             c   sr   d|_ d|j|j _| jd7  _| jttddd}t|V  t	|jdk sRt
tdd |j D snt
d S )NFrR   rQ   g{Gz?)r   c             s   s   | ]}|j d kV  qdS )r   N)r   )rS   r   r-   r-   r.   rU   (  s    z+test_occupancy_cleardown.<locals>.<genexpr>)r   rC   r1   r   r  r   r#   rY   r   absr3   r   r6   )rD   r9   r:   r;   r   r-   r-   r.   test_occupancy_cleardown  s    
r
     c             g   sB   | j ttddd}t|V  dd |j D ddhks>td S )Nr~   g?)r   c             S   s   h | ]}t |jqS r-   )r8   r   )rS   r   r-   r-   r.   r   0  s    z,test_balance_many_workers.<locals>.<setcomp>r   r=   )r   r#   rY   r   rC   r6   r3   )rD   r9   rC   r   r-   r-   r.   test_balance_many_workers+  s    
r  c             g   sN   d|j d j_| jttddd}t|V  dd |j D dhksJt	d S )	Ni ZstealingZ   g?)r   c             S   s   h | ]}t |jqS r-   )r8   r   )rS   r   r-   r-   r.   r   9  s    z.test_balance_many_workers_2.<locals>.<setcomp>r\   )

extensionsZ_pcZcallback_timer   r#   rY   r   rC   r6   r3   )rD   r9   rC   r   r-   r-   r.   test_balance_many_workers_23  s    
r  c             c   sf   | j tdd|jd}tdV  | jttddd}t|V  tdd |j	
 D rZt|  d S )	Nr=   g?)r   rC   g?rQ   )r   c             s   s&   | ]}|j  D ]}|d kV  qqdS )g      ?N)r   r6   )rS   r   vr-   r-   r.   rU   D  s    z8test_learn_occupancy_multiple_workers.<locals>.<genexpr>)r]   r#   r1   r   r	   r   rY   r   r  rC   r6   r3   r   )rD   r9   r:   r;   rE   r   r-   r-   r.   %test_learn_occupancy_multiple_workers<  s    
r  c             c   s   d|j d< | jtdtt|jd}| jtdttd |jd}| jt||dd}x*|j|jksl|j|j j	szt
d	V  qRW |j|j }|j	|j|j kst|j|j j| dkstt|V  ~d S )
NgMbP?r$      0)rC      1g      ?r=   )r   g{Gz?)r   r]   r   intr   r1   r$   rA   r@   Zprocessing_onr   r	   rC   r3   r   r   )rD   r9   r:   r;   rE   rF   rH   r   r-   r-   r.   'test_include_communication_in_occupancyI  s    

r  c             c   s   t tddd}t t|dd}t t|dd}| ||g\}}x(tdd |j D sjtdV  qDW t|j	|j
dd}	|	|jd |	 V  t }
xt|jdk rtdV  qW |j|j|j|jhd	|jd
|jd|jdikst|	 V  d S )Nr=   g?)r   c             s   s   | ]}|j V  qd S )N)r   )rS   r   r-   r-   r.   rU   b  s    z;test_worker_arrives_with_processing_data.<locals>.<genexpr>g{Gz?)r5   r\   )rp   releasedmemoryr   )r
   r#   r?   r  rC   r6   r   r	   r   r   r   put_key_in_memoryrA   r   r   r8   get_task_statusr3   r   )rD   r9   r:   r;   rE   rF   rH   rM   rN   r   r   r-   r-   r.   (test_worker_arrives_with_processing_dataZ  s"    
r  c             c   s   | j tddd}x tdD ]}| j t|dd}qW t|V  |jj  tdV  t	 }t|ddV  t	 }|| dk s~t
tdd |j D }|dddkst
d S )	Nr=   g?)r   r   )r   c             s   s   | ]}|j V  qd S )N)r   )rS   r   r-   r-   r.   rU     s    z1test_worker_breaks_and_returns.<locals>.<genexpr>)r  r  )r]   r#   rY   r   Zbatched_streamr   r   r   r	   r   r3   r   r@   r6   )rD   r9   r:   r   r|   r   endZstatesr-   r-   r.   test_worker_breaks_and_returnsx  s    
r  c       	      c   s   t tddd}t t|dd}t t|dd}| ||g\}}x|jsXtdV  qDW t|j|jdd}|	|j
d | V  t }x|jstdV  qW |j|j
|j
|j
hd|j
d|j
d	|j
d
ikst| V  d S )Nr=   g?)r   g{Gz?)r5   r\   )rp   r  r  r   )r
   r#   r?   r@   r   r	   r   r   r   r  rA   r   r   rC   r  r3   r   )	rD   r9   rE   rF   rH   rM   rN   r   r   r-   r-   r.   test_no_workers_to_memory  s"    
r  c       
      c   s   t tddd}t t|dd}t t|dd}| j||g|||fdid\}}x|jsftdV  qRW t|j|jddd}	|		|j
d |	 V  xt|jdk rtdV  qW td	V  |j|j
|j
|j
hd
|j
d|j
d|j
dikst|	 V  d S )Nr=   g?)r   r   )rC   g{Gz?)r5   r   r\   g333333?)rp   r  r  r   )r
   r#   r?   r@   r   r	   r   r   r   r  rA   r   r8   rC   r  r3   r   )
rD   r9   r:   r;   rE   rF   rH   rM   rN   r   r-   r-   r.   %test_no_worker_to_memory_restrictions  s"     
r  c             C   s   ddd}t  h\}\}}t|d | dD}||}||d ksFttt |tdd W d Q R X W d Q R X W d Q R X d S )Nc             S   s   | j S )N)r1   )dask_schedulerr-   r-   r.   rG     s    z%test_run_on_scheduler_sync.<locals>.fr1   )r*   r=   r   )N)r&   r   Zrun_on_schedulerr3   rd   re   ZeroDivisionErrorr'   )r*   rG   r9   r:   r;   rD   r1   r-   r-   r.   test_run_on_scheduler_sync  s    

r!  c             c   s(   ddd}|  |V }||jks$td S )Nc             S   s   | j S )N)r1   )r  r-   r-   r.   rG     s    z test_run_on_scheduler.<locals>.f)N)Z_run_on_schedulerr1   r3   )rD   r9   r:   r;   rG   r   r-   r-   r.   test_run_on_scheduler  s    
r"  c             c   sf   t |jdkst|j|jdV  t |jdks4t|j|jksDttdV  t |jdksbtd S )NrR   )workerr=   g      ?)r8   rC   r3   close_workerr1   r   r	   )rD   r9   r:   r;   r-   r-   r.   test_close_worker  s    r%  r~   )r>   r   r   c             c   s  t |jdkst|j s t|j}t }|j|dV  t |jdksLt||jksZtt }x*| rt	dV  t |d k sbtqbW |j
d kstxHtdD ]<}t	dV  t |jdkst| rt|j
d kstqW x.|jdkrt	dV  t |d k stqW d S )	NrR   )r#  r=   g?r   r   closedg?)r8   rC   r3   Zprocessis_aliver   r   r$  r   r	   pidrY   r   )rD   r9   r:   r;   Za_worker_addressr   r|   r-   r-   r.   test_close_nanny  s(    
r)  c             c   sB   |j ddV  |jrtx$|jdkr<|jdkr<tdV  qW d S )NT)r   r&  g{Gz?)rl   rC   r3   r   r   r	   )rD   r9   r:   r;   r-   r-   r.   test_retire_workers_close  s    
r*  )r>   r   r   c             c   s   ||g}|j dddV  |jr"tt }x4tdd |D r\tdV  t |d k s*tq*W tdd |D rtt|jr~td S )NT)r   removec             s   s   | ]}|j d kV  qdS )r&  N)r   )rS   r   r-   r-   r.   rU     s    z,test_retire_nannies_close.<locals>.<genexpr>g?r   c             s   s   | ]}|  V  qd S )N)r'  )rS   r   r-   r-   r.   rU     s    )rl   rC   r3   r   r  r   r	   )rD   r9   r:   r;   r  r   r-   r-   r.   test_retire_nannies_close  s    
r,  )z	127.0.0.1rR   c             c   sh   g }x>t dD ]2}| jt|dd| dd}|| tdV  qW t|d V  |d jd	ksdtd S )
Nr~   g?zinc-%02dg{Gz?)r   rA   Zfifo_timeoutg{Gz?r   Zfinished)	rY   r]   r#   appendr   r	   r   r   r3   )rD   r9   r   r   r|   r   r-   r-   r.   test_fifo_submission#  s    
r/  c           
   c   sv   t  \} t| d}|d t| }t|}W d Q R X |d |jksLtt| |j	ddV }W d Q R X |
 V  d S )N)scheduler_filer   r1   T)r0  r*   Zasynchronous)r   r   r   r  jsonloadr1   r3   r   r*   r   )fnr9   rG   rK   rD   r-   r-   r.   test_scheduler_file/  s    


r4   c          	   c   st   t jddiZ |jddi i d | ttd}tdV  |j	rJt
tdd	 |j D sft
W d Q R X d S )
Nz!distributed.comm.timeouts.connectr   z127.0.0.1:5738rR   )r1   r5   nbytesr   r   g333333?c             s   s   | ]}|j d kV  qdS )z	no-workerN)r   )rS   r   r-   r-   r.   rU   D  s    z+test_non_existent_worker.<locals>.<genexpr>)r   r   r   r   r   r   rY   r   r	   rC   r3   r   r@   r6   )rD   r9   r   r-   r-   r.   test_non_existent_worker<  s    
r7  c             '   sp    j tdddtV   fddtdD }tdV  t|V  tdd	 |D sltd
d |D d S )Nr=   r   )r   c                s   g | ]} j td ddqS )g?F)r   pure)r]   r#   )rS   r|   )rD   r   r-   r.   rT   L  s   z2test_correct_bad_time_estimate.<locals>.<listcomp>r~   g      ?c             s   s   | ]}|j V  qd S )N)rK   )rS   r   r-   r-   r.   rU   S  s    z1test_correct_bad_time_estimate.<locals>.<genexpr>c             S   s   g | ]}t |jqS r-   )sortedrK   )rS   r   r-   r-   r.   rT   S  s    )r]   r#   r   rY   r   r	   r   r3   )rD   r9   rC   r   r-   )rD   r   r.   test_correct_bad_time_estimateG  s    

r:  Zlinuxz Need 127.0.0.* to mean localhostc              c   s   t d ddlm}  xdD ]}xdD ]\}}d|f| i}t|d}||V  t|jd jj	j
 }t|tr| d |d kstn4t|tr| d |kstn| d |kst| V  q&W qW d S )NZbokehr   )BokehScheduler)r   )z	127.0.0.3r   ))ztcp://0.0.0.0)z::z0.0.0.0)ztcp://127.0.0.2z	127.0.0.2)ztcp://127.0.0.2:38275z	127.0.0.2)r   )rd   r   Zdistributed.bokeh.schedulerr;  r   r   r   r   ZserverZ_httpZ_socketsr6   r0   tupleZgetsocknamer3   r   )r;  r   Zurlr   r   r9   Zsockr-   r-   r.   test_service_hostsV  s    




r=  profile_cycle_intervalrQ   )r>   worker_kwargsc             #   s   t  d | jttdd|jd}t|V  tdV  |jddV }t  d  |sXt	t
 fdd	|d
 D sxt	t
dd	 |d
 d d D st	|d
 d d rt	d S )Nr=   r   g?)r   rC   g?g?)r>  c             3   s*   | ]"\}}|  k o k n  V  qd S )Nr-   )rS   tcount)nowr   r-   r.   rU   z  s    z(test_profile_metadata.<locals>.<genexpr>countsc             s   s*   | ]"\}}d |  kodk n  V  qdS )r   r  Nr-   )rS   r@  rA  r-   r-   r.   rU   {  s    r   r-  )r   r   r#   rY   r1   r   r   r	   get_profile_metadatar3   r   )rD   r9   r:   r;   r   metar-   )rB  r   r.   test_profile_metadatap  s    


 "rF  c             c   s   t  d }| jttddd}| jttddd}t|| V  |jddV }t|d dd	hksftt	|d
 t	|d d kstd S )Nr=   r   g?)r   g?)r>  rp   r#   r%   rC  )
r   r   r#   rY   r%   r   rD  r   r3   r8   )rD   r9   r:   r;   r   rE   rF   rE  r-   r-   r.   test_profile_metadata_keys  s    
rG  c       	      c   s   t tddd}t t|dd}t t|dd}t t|dd}| |}t| tdV  |jddV  |jdkszt|j	rtd S )Nr=   g?)r   T)ZforceZ	cancelled)
r
   r#   rL   r   r   r	   cancelr   r3   r@   )	rD   r9   r:   r;   rE   rF   rH   r   r   r-   r-   r.   test_cancel_fire_and_forget  s    
rI  c             c   s.   |  tjd}t|V  dt|jks*td S )Nr   exit)r]   sysrJ  r   r2   r   r3   )rD   r9   r:   r;   r   r-   r-   r.   test_log_tasks_during_restart  s    
rL  rR   c             #   s   | j tdddV  | jttddd}| jttddd jd}x*t|jt|t| k rjtdV  qBW x|D ]}|j	|j
d	 qrW t|V  tfd
d|D dkstt fdd|D dkstd S )Nr-  g?)r   r   r   r~   )r   rC   gMbP?)rA   c             3   s   | ]}|j  jkV  qd S )N)rA   rK   )rS   r   )r;   r-   r.   rU     s    z"test_reschedule.<locals>.<genexpr>r\   c             3   s   | ]}|j  jkV  qd S )N)rA   rK   )rS   r   )r:   r-   r.   rU     s    r=   )r]   r#   r   rY   r1   r8   r@   r   r	   Z
reschedulerA   r   r4   r3   )rD   r9   r:   r;   rE   r   r   r-   )r:   r;   r.   test_reschedule  s    

rM  c             c   s@   |  td}t|V  |jj|jgdV }||jdiks<td S )Nr=   )rp   r  )r]   r   r   rv   r  rA   r3   )rD   r9   r:   r;   r   rO   r-   r-   r.   test_get_task_status  s    
rN  c              C   sl   ddl m}  t }|j}| d t|jdks4t|jd }d||ksPtt	dd |jD shtd S )	Nr   )loggerfoo123r=   r-  zdistributed.schedulerc             s   s   | ]}|j d kV  qdS )rP  N)r   )rS   r   r-   r-   r.   rU     s    z%test_deque_handler.<locals>.<genexpr>)
distributed.schedulerrO  r   Z_deque_handlerinfor8   dequer3   formatr  )rO  r9   Zdeque_handlerr   r-   r-   r.   test_deque_handler  s    

rU  c       	   	   c   s   t dt ddg}| jt|dd}|V }|dks6t|j|j jdksLt|j|jks\t| jt|ddd	}|V }|dkst|j|j jd
kst|j|jkst| jt|ddd	}t	t }|V }W d Q R X |
d | jt|d
dd	}t	t }|V }W d Q R X |
d d S )NZoneZtwo*   r\   )retriesr=   rR   F)rW  r8  r   )r   r]   r(   r3   r@   rA   rW  
exceptionsrd   re   match)	rD   r9   r:   r;   r   r   rO   exc_infoZresr-   r-   r.   test_retries  s&    
r[  z)second worker also errant for some reasonr   c          	   c   s   t jddi td}| j|jjd|jd}t|V  | j	||j|jgdV  | jt
||jd}x|js|tdV  qhW |  t|V  W d Q R X d S )Nz!distributed.comm.timeouts.connectZ1sr   i )rC   gMbP?)r   r   r   rd   r   r]   r   r1   r   Z	replicater8   r@   r   r	   r   )rD   r9   Zw1r   Zw3r   rE   rF   r-   r-   r.   test_mising_data_errant_worker  s    

r\  c       	      c   sp   t tddd}t t|dd}| }t|V  t|j}| }t|V  tdV  t|j|ksltd S )Nr=   rE   )dask_key_namerF   g?)	r
   r   r?   r   rX   r   r   r	   r3   )	rD   r9   r:   r;   rE   rF   rM   oldyyyr-   r-   r.    test_dont_recompute_if_persisted  s    


r`  c       
      c   s   t tddd}t t|dd}t t|dd}| }t|V  |dd}| }	t|	V  tdV  |dd|kstd S )Nr=   rE   )r]  rF   rH   g?)r
   r   r?   r   r   r   r	   r3   )
rD   r9   r:   r;   rE   rF   rH   rM   r^  rN   r-   r-   r.   "test_dont_recompute_if_persisted_2  s    

ra  c             c   s   t tddd}t tddd}t t|dd}t t||dd}| }t|V  t|j}	| }
t|
V  tdV  t|j|	kst	d S )	Nr=   rE   )r]  rR   rF   rH   r   g?)
r
   r   r   r?   r   rX   r   r   r	   r3   )rD   r9   r:   r;   rE   rF   rH   r   Zwwr^  Zwwwr-   r-   r.   "test_dont_recompute_if_persisted_3  s    


rb  c             c   s   t tddd}t t|dd}t t|dd}| }t|V  |d}x |jd jdkrltdV  qNW t	||\}	}
t|	|
gV  |d}t
|t
|kstd S )Nr=   rE   )r]  rF   rH   r  g{Gz?)r
   r   r?   r   r   r@   r   r   r	   r   r8   r3   )rD   r9   r:   r;   rE   rF   rH   rM   r^  r_  Zzzznewr-   r-   r.   "test_dont_recompute_if_persisted_4*  s    


rd  c             c   sp   | j tddd}| j t|dd}| j t|dd}~t||gV  ~xd|jkr\tdV  qDW d|jksltd S )Nr=   rE   )rA   rF   rH   g{Gz?)r]   r   r    r   r@   r   r	   r3   )rD   r9   r:   r;   rE   rF   rH   r-   r-   r.   test_dont_forget_released_keys?  s    re  c       	      c   sr   t tddd}t t|ddd}| }t|V  t|j}| }t|V  tdV  t|j|ksnt	d S )Nr=   rE   )r]  r   rF   g?)
r
   r   r'   r?   r   rX   r   r   r	   r3   )	rD   r9   r:   r;   rE   rF   rM   r^  r_  r-   r-   r.   test_dont_recompute_if_erredN  s    


rf  c             c   sL   |   V  t }x6|jdks&|jdkrFtdV  t |d k stqW d S )Nr&  g{Gz?rR   )r   r   r   r   r	   r3   )r9   r:   r;   r   r-   r-   r.   %test_closing_scheduler_closes_workers_  s
    
rg  	resourcesAr=   )r>   r5   r?  c             c   s   | j tdddid}x|js*tdV  qW | V  x|jrLtdV  q8W |j|j jd rdt|j	ddiksvt| j t
dddidV  d S )Ng?ri  r=   )rh  g{Gz?)r]   r	   Z	executingr   rH  rC   r1   Zused_resourcesr3   Zavailable_resourcesr   )rD   r9   r   r   r-   r-   r.   )test_resources_reset_after_cancelled_taski  s    
rj  c             c   st   dd }dd }dd }| j |dd}| j ||d	d}|V  | j ||d
d}	~tdV  | j ||d	d}
|
V  d S )Nc               S   s   dS )Nfoor-   r-   r-   r-   r.   rk  ~  s    ztest_gh2187.<locals>.fooc             S   s   | d S )Nbarr-   )rE   r-   r-   r.   rl    s    ztest_gh2187.<locals>.barc             S   s   t d | d S )Ng?baz)r	   )rE   r-   r-   r.   rm    s    ztest_gh2187.<locals>.bazrE   )rA   rF   rH   g?)r]   r   r	   )rD   r9   r:   r;   rk  rl  rm  rE   rF   rH   rG   r-   r-   r.   test_gh2187|  s    rn  )Z
__future__r   r   r   r   collectionsr   Zdatetimer   r1  operatorr   r   rK  r   r	   r   r
   Ztoolzr   r   r   r   r   Ztornador   rd   Zdistributedr   r   r   r   r   Zdistributed.corer   r   rQ  r   r   Zdistributed.clientZdistributed.metricsZdistributed.protocol.pickler   Zdistributed.workerr   r   Zdistributed.utilsr   Zdistributed.utils_testr   r    r!   r"   r#   r$   r%   r&   r'   r(   r)   r*   r+   Zdask.compatibilityr,   r   Zbobr   r<   rJ   rP   r[   r^   r_   r`   rh   rj   rk   rm   rs   r{   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   ZmarkZskipifplatformry   r   r   r   r   r   r   r   r   r   r   r   r   version_infor  r  r  r
  r  r  r  r  r  r  r  r  r!  r"  r%  r)  r*  r,  r/  r4  Zxfailr7  r:  r=  rF  rG  rI  rL  rM  rN  rU  r[  r\  r`  ra  rb  rd  re  rf  rg  rj  rn  r-   r-   r-   r.   <module>   s  4
	!#	!	4		
	
