B
    ^\RQ                @   s6!  d dl mZmZmZ d dlmZ d dlmZ d dlm	Z	 d dl
Z
d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlmZ d dlmZ d dlZd dlZd dlZd dlZd dlZd dlmZmZmZmZmZm Z m!Z!m"Z" d d	l#m$Z$ d d
l%m&Z& d dl'Z'd dl'm(Z( d dl)m*Z* d dl+m,Z- d dl.m/Z/m0Z0m1Z1m2Z2m3Z3m4Z4m5Z5m6Z6m7Z7m8Z8 d dl9m:Z: d dl;m<Z<m=Z=m>Z>m?Z?m@Z@mAZAmBZBmCZCmDZD d dlEmFZFmGZG d dlHmZ d dlImJZJmKZK d dlLmMZM d dlNmOZOmPZPmQZQmRZRmSZSmTZT d dlUmVZVmWZWmXZXmYZYmZZZm[Z[m\Z\m]Z]m^Z^m_Z_m`Z`maZambZbmcZcmdZdmeZemfZfmgZgmhZhmiZimjZjmkZk d dlUmlZmmnZompZpmqZqmrZrmsZsmtZtmuZumvZv ebddddd Zwebdddd Zxebdddd Zyebddd d! Zzebddd"d# Z{ebddd$d% Z|d&d' Z}ebddd(d) Z~ebddd*d+ Zebddd,d- Zebddd.d/ Zebddd0d1 Zebddd2d3 Zd4d5 Zebddd6d7 Zd8d9 Zd:d; Zebddd<d= Zebddd>d? Zebddd@dA Zeb dBdC ZdDdE ZdFdG ZebdddHdI ZebdddJdK ZdLdM ZebdddNdO ZebddPgdQdRdS ZebdddTdU ZebddddVdW ZdXdY ZdZd[ Zd\d] Zebddd^d_ Zebddd`da Zebddbddcdd Zdedf Zdgdh Zebdddidj Zebdddkdl Zebdmddndodp ZeWebdddqdr Zejjebdddsdt Zejjebdddudv Zejjebdddwdx Zebdddydz Zejjejd{ d|d}ebdPd~gdddd Zebdddd Zejjejd{ d|d}ebdPd~gdddd Zejjejd{ d|d}ebdPd~gdddd Zebdddd Zebddddd ZebdPgdddd Zebdddd Zebdddd Zeb dd Zebdddd Zebdddd Zebdddd Zebdddd Zebdddd Zebdddd Zebdddd Zebdddd Zebdddd Zebdddd Zebdddd Zebdddd Zdd Zdd Zebdddd Zebdddd Zejjejd{ d|d}ebdPd~gdddd Zebdddd Zebdddd Zebdddd Zebdddd Zdd Zebdddd Zdd ZddĄ ZddƄ ZddȄ Zddʄ Zejjdd}dd̈́ Zebddddτ Zebddddф Zebddddӄ ZebddPgd dQddք ZebddPgd dQdd؄ Zebdg dQddڄ ZebdddPgd d܍ddބ Zejdddgejdddgdd Zebdddd Zebdddd Zebdddd Zebdddd Zebdddd Zdd Zebdddd Zebdddd Zebdddd Zebdddd Zdd Zebdddd Zdd Zejjeb dd Zebddd d Zebdddd Zdd Zebdddd Zebdmddndd	 Zd
d Zebdddd Zejjejd{ d|d}ebdPd~gdddd Zejjddd}dd Zebdddd Zebdddd ZG dd deZG dd deZebdddd Zejjddd}dd  Zd!d" Zebddd#d$ Zec d%d& Zebddd'd( Zebddd)d* Zebddd+d, Zebddd-d. Zd/d0 Zebddd1d2 Zeb d3d4 Z d5d6 Zeb d7d8 Zeb d9d: Zebddd;d< ZebddPgd dQd=d> Zebddd?d@ ZdAdB ZebdddCdD ZebdddEdF Z	ebdddGdH Z
eb dIdJ ZebdddKdL ZdMdN ZebdddOdP ZebdddQdR ZebdddSdT ZejjejdUkdVd}ebdddWdX ZebdddYdZ Zebddd[d\ Zd]d^ Zebddd_d` Zdadb Zebdcddnddde Zebdddfdg Zdhdi Zebdcddndjdk Zebdddldm Zebdddndo Zdpdq Zebdddrds ZejjebdPgdddtdu Z ebg dddvdw Z!ejjdxd}ebdddydz Z"ebddd{d| Z#ebddd}d~ Z$dd Z%ebdddd Z&dd Z'dd Z(dd Z)dd Z*ebdddd Z+ec dd Z,dd Z-ebdddd Z.ejjejddd}dd Z/eb dd Z0dd Z1ebdddd Z2ebdddd Z3ebdPgd dddd Z4ebdddd Z5dd Z6ebdddd Z7ebdddd Z8ejjejd{ d|d}ebdPd~gdddd Z9ebdg dQdd Z:ejjejd{ d|d}ebdPd~gdddd Z;ebdddd Z<ebddPgd dQdd Z=ebdddd Z>ebddPgd dQdd Z?G dd deZ@ebddPgd dQdd ZAebddPgd dQdd ZBejjejd{ d|d}ebddPddgdddd ZCddÄ ZDejjejddd}ebddgd dQdǐdȄ ZEebddPgdb dQdɐdʄ ZFebddPgdb dQdːd̄ ZGejjebddgdb dQdΐdτ ZHebddPgd dQdАdф ZIebddPgd dQdҐdӄ ZJebddgdb dQdՐdք ZKebddgdb dQdאd؄ ZLebddPgd dQdِdڄ ZMdېd܄ ZNebdddݐdބ ZOebdddߐd ZPdd ZQebddPgd dQdd ZReWebde0dddd ZSdd ZTdd ZUesdd ZVeWebe0dddd ZWdd ZXdd ZYdd ZZejאdddgdd Z[dd Z\ec dd Z]ebdddd  Z^ebdg dQdd Z_eWdd Z`eWejjejddd}ejjejd  dbkdd}ejjadd}ejאde/ddfe0dd	fgd
d Zbebddddd Zcdd Zddd Zedd Zfdd Zgebdddd ZhebddPgd dQdd Ziebdddd Zjebdddd Zkebdddd Zlebddd d! Zmebddd"d# ZnesebdPgd ddd$d% ZoebdPgd ddd&d' Zpebddd(d) Zqd*d+ Zrebddd,d- Zsebddd.d/ ZtebddPgd dQd0d1 Zuebdd2gdb dQd3d4 ZvebddPd5gdQd6d7 Zwebddd8d9d: Zxebddd;d< Zyebddd=d> ZzeWd?d@ Z{ebdddAdB Z|dCdD Z}dEdF Z~ebddddGdH ZejjdId}ebddddJdK ZebdddLdM ZeWec dNdO ZebdddPdQ ZebdddRdS ZebdddTdU ZebdddVdW ZebdddXdY ZebdddZd[ Zebddd\d] Zebddd^d_ Zebddd`da Zdbdc Zddde Zebdddfdg ZG dhdi dieZebdddjdk Zebdddldm Zebdddndo Zebdddpdq Zebdddrds Zdtdu Zdvdw Zeb dxdy Zdzd{ ZeWd|d} ZeWd~d Zebdddd Zebddgdb dQdd Zdd Zebdddd ZebddPgd dddd ZeWebddPgdb dcddd Zebdddd Zdd Zebdddd Zdd ZdddZdd Zdd ZeWdd Zebdddd Zebdddd Zebdddd Zdd Zdd Zdd Zebdgdb dddd Zebdgdb dddd Zebdgdb dddd Zebdgdb dddd Zebdddiddd Zebdddiddd Zeb dd Zebdddd Zebdddd Zdd ZeWebdeFre0ne/dcd2gdb ddd ZebddddÄ ZebdddĐdń ZebdddƐdǄ Zeb dȐdɄ Zebdddʐd˄ Zeb ddτ Zejjejd  dk dd}ebdddΐdτ Zeb dАdф ZdҐdӄ Zebdg dQdԐdՄ Zejjdejkdd}dؐdل Zec dڐdۄ Zec dܐd݄ Zejdddgdސd߄ Zeb dd Zeb dd Zeb dd Zːdd ZebddPgdQdd Zebdddd Zΐdd Zeb dd ZАdd Zeb dd Zebdddd Zebdddd Zebdddd Zebdddd Zebdddd Zebddd8dd Zebddd d Zebdddd Zejdk!r2d dlT dS (      )print_functiondivisionabsolute_import)add)deque)CancelledErrorN)	Semaphore)sleep)identity
isdistinctconcatpluckvalmappartialfirstmerge)gen)IOLoop)delayed)SubgraphCallable)
WorkerNannyfire_and_forgetLocalCluster
get_clientsecede
get_workerExecutorprofileTimeoutError)CommClosedError)	ClientFuturewaitas_completedtokenize_get_global_clientdefault_client
futures_oftemp_default_client)PY3Iterator)time)	SchedulerKilledWorker)sizeof)ignoring
mp_contextsynctmp_texttokeytmpfile)clusterslowslowincslowaddslowdec	randomincincdecdivthrowsgenincasyncincgen_clustergen_testdoublepopencaptured_loggervaryingmap_varyingwait_forasync_wait_forpristine_loop)	clientclient_secondarycluster_fixturelooploop_in_threadnodebugsabT)rL   timeoutc             c   s   |  td}| rtt|ts&t|j| ks4t|V }|dksFt| sRt|  td}|  t||}|V }|dks~t|  d S )N
             )	submitr<   doneAssertionError
isinstancer"   rL   r   validate_state)crR   rS   rT   xresultyz rd   <lib/python3.7/site-packages/distributed/tests/test_client.pytest_submit9   s    rf   )rL   c             c   s  |  ttd}t|dks ttdd |D s6ttdd |D sLt|d V }|tdksftt|jdksxt|  t|}|d V }|ttdkstt|jdkst| t	|}|V }|t	t tt ttdkst|  t
||}|d V }|tdttd kst|  t
tdtd	}	| |	V }
tjd dkrp|
tt t
tdtd	ksptdd
d}| j |tddd}| |V }
|
ttddkst| |d}| j |td|d}| |V }
|
ttddkst|  d S )N   c             s   s   | ]}|j V  qd S )N)key).0r`   rd   rd   re   	<genexpr>Q   s    ztest_map.<locals>.<genexpr>c             s   s   | ]}t |tV  qd S )N)r]   r"   )ri   r`   rd   rd   re   rj   R   s    r      rV         c             S   s   | | S )Nrd   )r`   rb   rd   rd   re   fl   s    ztest_map.<locals>.f)rb   rX      )rV   )mapr<   rangelenr\   r   alltasksrZ   sumr   gathersysversion_infolistr^   )r_   rR   rS   rT   L1ra   L2totalL3L4resultsrn   ZL5rb   ZL6rd   rd   re   test_mapM   s<    

 
"
r   c             c   s<   | j tg dd}t|dks t| |V }|g ks8td S )NF)purer   )rp   r<   rr   r\   rv   )r_   rR   rS   rT   rz   r   rd   rd   re   test_map_emptyz   s    r   c             C   s   | j ttddd}tdd |D s*ttdd |D s@t| j tddd	d
gdd}dd |D dd |D ksxtddddg}| j ttd|d}dd |D |kstd S )Nrm   INC)rh   c             s   s   | ]}|j d V  qdS )r   N)rh   
startswith)ri   rn   rd   rd   re   rj      s    z$test_map_keynames.<locals>.<genexpr>c             s   s   | ]}|j V  qd S )N)rh   )ri   rn   rd   rd   re   rj      s    rg            c             S   s   g | ]
}|j qS rd   )rh   )ri   rn   rd   rd   re   
<listcomp>   s    z%test_map_keynames.<locals>.<listcomp>zinc-1zinc-2zinc-3zinc-4c             S   s   g | ]
}|j qS rd   )rh   )ri   rn   rd   rd   re   r      s    )rp   r<   rq   rs   r\   r   )r_   rR   rS   rT   futuresfutures2keysrd   rd   re   test_map_keynames   s     r   c       	   	   c   sF  t dddgdddgt dt dd	gg}| jt|d
di\}}}|V dksPt|V dks^t|V d	kslt| jt|ddd\}}}|V dkst|V dksttt }|V  W d Q R X |d | jt|ddd\}}}tt }|V  W d Q R X |d |V dksttt }|V  W d Q R X |d d S )None   rl   rm   rg   r   ZsevenZeight	   retriesrk   F)r   r   r   )ZeroDivisionErrorrp   rH   r\   pytestraisesmatch)	r_   rR   rS   rT   argsr`   rb   rc   exc_inford   rd   re   test_map_retries   s*    

r   c          	   c   s  t dt ddg}| tt| }tt }|V  W d Q R X |d | jtt| dd}tt }|V  W d Q R X |d | jtt| dd}|V dkst|d | jtt| dd}|V dkstt dt ddd	g}t d
t ddg}dddg}	dd ||fD \}}
| j||
g|did\}}
t	
  |V dksXttt }|
V  W d Q R X |d
 dd |||	fD \}}
}| j||
|g|
|fdid\}}
}tt }|V  W d Q R X |d |
V dkst|V dks td S )Nr   tworl   rk   )r   r   rm      (   fivesixF   P   Z   d   c             S   s   g | ]}t t| qS rd   )r   rG   )ri   r   rd   rd   re   r      s    z(test_compute_retries.<locals>.<listcomp>c             S   s   g | ]}t t| qS rd   )r   rG   )ri   r   rd   rd   re   r      s    )r   computer   rG   r   r   r   r\   appendgccollect)r_   rR   rS   rT   r   r`   r   xargsyargszargsrb   rc   rd   rd   re   test_compute_retries   s>    




"
r   c          	   C   sv   t dt ddg}tt| }|jdddks4tt dt ddg}tt| }tt  |  W d Q R X d S )Nr   r   rl   rg   )r   )r   r   rG   r   r\   r   r   )r_   r   r`   rd   rd   re   test_retries_get   s    r   c          	   c   s  t dt ddg}| tt| }| |}tt }|V  W d Q R X |d | tt| }| j|dd}tt }|V  W d Q R X |d | tt| }| j|dd}|V dkst|	d | tt| }| j|dd}|V dks
td S )Nr   r   rl   rk   )r   r   rm   )
r   persistr   rG   r   r   r   r   r\   r   )r_   rR   rS   rT   r   r`   Zfutr   rd   rd   re   test_compute_persisted_retries   s$    



r   c          	   c   sV  t dt ddg}| jtt| dd}| |}tt }|V  W d Q R X |d | jtt| dd}| |}|V dkstt dt dddg}t d	t d
dg}dddg}	dd |||	fD \}}
}| j||
|g|
|fdid\}}
}| ||
|g\}}
}tt }|V  W d Q R X |d |
V dksBt|V dksRtd S )Nr   r   rl   rk   )r   r   r   r   r   r   r   r   r   r   c             S   s   g | ]}t t| qS rd   )r   rG   )ri   r   rd   rd   re   r     s    z(test_persist_retries.<locals>.<listcomp>)	r   r   r   rG   r   r   r   r   r\   )r_   rR   rS   rT   r   r`   r   r   r   r   rb   rc   rd   rd   re   test_persist_retries  s(    



"
r   c             c   s@   t d}|jddd}| j| dd}|V }|dks<td S )Nz
dask.array)rV   rV   )rl   rl   )chunksr   )r   r   )r   importorskiponesr   ru   r\   )r_   rR   rS   rT   dar`   futurerb   rd   rd   re   test_retries_dask_array   s
    
r   c             C   sp   |  td}x^tdd gD ]N}t|j||ks4tt|j||ksJtt|jt| j|j kstqW d S )NrV   c             S   s   |   S )N)_repr_html_)r`   rd   rd   re   <lambda>,  s    z"test_future_repr.<locals>.<lambda>)rZ   r<   reprstrrh   r\   statusr   )r_   rR   rS   rT   r`   funcrd   rd   re   test_future_repr)  s
    r   c       	      C   sf   t d}|jddd }t|d }x8tdd gD ](}x"|jD ]}t|||ksBtqBW q6W d S )Nz
dask.arrayrV   )rg   )r   r   c             S   s   |   S )N)r   )r`   rd   rd   re   r   7  s    z(test_future_tuple_repr.<locals>.<lambda>)	r   r   aranger   r(   r   rh   r   r\   )	r_   rR   rS   rT   r   rb   rn   r   krd   rd   re   test_future_tuple_repr2  s    
r   c             c   sN   |  tdd}| V }t|ts&t|  tdd}| V }|d ksJtd S )Nrk   r   )rZ   r>   	exceptionr]   r   r\   )r_   rR   rS   rT   r`   ra   rd   rd   re   test_Future_exception<  s    

r   c             C   sB   |  tdd}t| ts t|  tdd}| d ks>td S )Nrk   r   )rZ   r>   r]   r   r   r\   )r_   r`   rd   rd   re   test_Future_exception_syncG  s    r   c             c   s   |  tdd}|V  |  tjV  | jr.t| j tddd}|  tjV  | jrXt|  tdd}| V  |  tjV  | jrtd S )Nrk   g      ?)delayr   )	rZ   r>   releaser   momentr   r\   r8   r   )r_   rR   rS   rT   r`   rd   rd   re   test_Future_releaseO  s    


r   c                s     tdd}|  |  t fdddd  j tddd}|  t fdddd   tdd	}|  |  t fd
ddd d S )Nrk   c                  s    j  S )N)r   rd   )r_   rd   re   r   i  s    z*test_Future_release_sync.<locals>.<lambda>g333333?)rU   g?)r   c                  s    j  S )N)r   rd   )r_   rd   re   r   m  s    r   c                  s    j  S )N)r   rd   )r_   rd   re   r   r  s    )rZ   r>   ra   r   rI   r8   r   )r_   r`   rd   )r_   re   test_Future_release_syncd  s    r   c             C   s   t d}|tdd}y|  W n" tk
rF   t \}}}Y nX ||	 }d}x|d k	rv|d7 }|d }q\W |dk st
d S )Ntblibrk   r   tb_nextrg   )r   r   rZ   r>   ra   	Exceptionrw   r   Z	TracebackZto_dictr\   )rO   r_   r   r   _tbnrd   rd   re   test_short_tracebacksu  s    

r   c             C   s   |  ttd}|  ttd}dd |D dd |D ks@t|  tddddg}tdd |D dksnt| j tddddgdd}td	d |D d
kstd S )Nrg   c             S   s   g | ]
}|j qS rd   )rh   )ri   r`   rd   rd   re   r     s    z#test_map_naming.<locals>.<listcomp>rk   c             S   s   h | ]
}|j qS rd   )_state)ri   r`   rd   rd   re   	<setcomp>  s    z"test_map_naming.<locals>.<setcomp>F)r   c             S   s   h | ]
}|j qS rd   )r   )ri   r`   rd   rd   re   r     s    rm   )rp   r<   rq   r\   rr   )r_   rR   rS   rT   rz   r{   r}   r~   rd   rd   re   test_map_naming  s     r   c             C   sL   |  td}|  td}|j|jks(t| j tddd} | j|jksHtd S )Nrk   F)r   )rZ   r<   r   r\   rh   )r_   rR   rS   rT   rd   rd   re   test_submit_naming  s
    r   c          	   c   sn   |  tdd}|V }|dks t|  tdd}tt |V }W d Q R X |  tdd}|V }|dksjtd S )Nrk   r   g      ?r   rV   g      @)rZ   r>   r\   r   r   r   )r_   rR   rS   rT   r`   ra   rd   rd   re   test_exceptions  s    r   c             #   sh   t  j jfddV }|tdV   jj js:t	  t
 fddddV  | V  d S )NT)asynchronousrV   c                  s   j  jkp jj  j S )N)rh   rt   who_hasrd   )rR   r`   rd   re   r     s    ztest_gc.<locals>.<lambda>g333333?)rU   )r!   ipportrZ   r<   rt   rh   r   r\   __del__rJ   close)rR   rS   rT   r_   rd   )rR   r`   re   test_gc  s    r   c          	   C   sd   |  td}| dkst| j tddd}ttj |jdd W d Q R X | dks`td S )Nrk   r   g333333?)r   g{Gz?)rU   )	rZ   r<   ra   r\   r8   r   r   r   r   )r_   r`   rd   rd   re   test_thread  s    r   c             C   sx   |  tdd}| dkst|  tdd}y|  ds>tW n tk
rT   Y nX |  tdd}| dksttd S )NrV   r   rg   r   F)rZ   r>   ra   r\   r   )r_   r`   rb   rc   rd   rd   re   test_sync_exceptions  s    r   c             c   sx   |  td}|  t|}| |V }|dks0t| |gV }|dgksLt| ||gdV }|ddgdksttd S )NrV   rW   )r`   rb      )rZ   r<   rv   r\   )r_   rR   rS   rT   r`   rb   ra   rd   rd   re   test_gather  s    r   c          	   c   s\   | j dg|jdV \}| jtd|jd}| V  tt | ||gV }W d Q R X d S )Nrk   )workers)	scatteraddressrZ   r<   _closer   r   r   rv   )r_   rR   rS   rT   r`   rb   resrd   rd   re   test_gather_lost  s
    
r   c          	   C   st   |  td}| |dkst|  tdd}tt | ||g W d Q R X | j||gdd\}|dksptd S )Nrk   r   r   skip)errors)rZ   r<   rv   r\   r>   r   r   r   )r_   r`   rb   xxrd   rd   re   test_gather_sync  s    r   c          	   c   sh   |  tdd}|  tdd}tt | ||gV  W d Q R X | j||gddV \}|dksdtd S )Nr   rk   r   r   )r   )rZ   r>   r   r   r   rv   r\   )r_   rR   rS   rT   r`   rb   r   rd   rd   re   test_gather_strict  s    r   )z	127.0.0.1rk   )rL   ncoresc          
   c   s   | j tdddd}| j tddd}ttdB}ttd(}| j||gd	d
V }|dgksdtW d Q R X W d Q R X | rt| rtd S )Nrk   r   rV   )priorityg      ?)r   zdistributed.schedulerzdistributed.clientr   )r   r   )	rZ   r>   r8   rF   logging	getLoggerrv   r\   getvalue)r_   rR   rS   r`   rb   ZschedrL   Lrd   rd   re   test_gather_skip  s    "r   c             c   s6   |  ttd}|V }t|jt|j dk s2td S )Nr   )rp   r<   rq   rr   Zoutgoing_transfer_logr\   )r_   rR   rS   rT   r   r   rd   rd   re   test_limit_concurrent_gathering  s    r   c             c   s   | j dtdfiddd}t|ts&t|V }|dks8t| j dtdfidgdd}t|d tsdt|V }|dgksxt| j i g ddV }|g kst| j tdftdfdd	ddV }|d
kstd S )Nr`   rk   F)r2   r   r   )r`   rk   ))r`   rk   )r`   r   )r`   r   rl   )getr<   r]   r"   r\   )r_   rR   rS   rT   r   ra   r   rd   rd   re   test_get  s    r   c             C   s    |  dtdfiddkstd S )Nr`   rk   r   )r   r<   r\   )r_   rd   rd   re   test_get_sync.  s    r   c             C   sp   ddl m} | }| ttd}|| ~dd l}|  t }x&t	|rjt
d t |d k sFtqFW d S )Nr   )WeakSetrV   g{Gz?r   )weakrefr   rp   r<   rq   updater   r   r,   ry   r	   r\   )r_   r   Zwsr   r   startrd   rd   re   test_no_future_references2  s    

r   c             C   s*   t jdddt}tj| dd d S )NrV   rl   )npartitionsF)optimize_graph)dbrq   rp   r<   daskr   ru   )r_   bagrd   rd   re   +test_get_sync_optimize_graph_passes_through@  s    r   c          	   c   s~   dd }dd }|  |dd}|  |dd}tt | |V  W d Q R X tt | |V  W d Q R X | V  d S )Nc             S   s   t d S )N)	TypeError)rS   rT   rd   rd   re   rn   G  s    ztest_gather_errors.<locals>.fc             S   s   t d S )N)AttributeError)rS   rT   rd   rd   re   gJ  s    ztest_gather_errors.<locals>.grk   r   )rZ   r   r   r   rv   r   r   )r_   rR   rS   rT   rn   r   Zfuture_fZfuture_grd   rd   re   test_gather_errorsE  s    r   c       	      c   sz   |  td}|  td}|  td}t|||gV \}}||||hksJt|t ksXt|j|j  krpdksvn td S )Nrk   r   finished)rZ   r<   r#   r\   setr   )	r_   rR   rS   rT   r`   rb   rc   r[   not_donerd   rd   re   	test_waitW  s    r  r   c          	   c   s6   |  td}ttj t|ddV  W d Q R X d S )Ng333333?g{Gz?)rU   )rZ   r	   r   r   r   r   r#   )r_   rR   rS   rT   r   rd   rd   re   test_wait_timeoutd  s    r  c          	   C   s   |  td}|  td}t||g\}}|||hks8t|t ksFt|j|j  kr^dksdn t|  td}tt	j
 t|dd W d Q R X d S )Nrk   r   r  g333333?g{Gz?)rU   )rZ   r<   r#   r\   r  r   r	   r   r   r   r   )r_   r`   rb   r[   r  r   rd   rd   re   test_wait_synck  s    r  c          
   C   sn   |  td}|  td}yt|| W nB tk
rh } z$dt|ksHtdt|ksXtW d d }~X Y nX d S )Nrk   r   rU   ry   )rZ   r<   r#   r   r   r\   )r_   r`   rb   erd   rd   re   (test_wait_informative_error_for_timeoutsy  s    r  c       	      c   s   |  td}|  td}| j|j dks,t|  tjV  | j|j dksPt|  t|}|  tjV  |V }|dks~t|j}|  tjV  || jkstd S )Nrk   r   rl   )	rZ   r<   refcountrh   r\   r   r   r   r   )	r_   rR   rS   rT   r`   rb   rc   ra   Zykeyrd   rd   re   test_garbage_collection  s    r
  c             c   s   |  dgV \}|j| jks t|jdks.t|j|j | jhksFt|j}| j| dks^t|  t	j
V  | j| dkstt }x<||jks|j| jsP qt |d k stt	dV  qW d S )Nrk   r  r   rl   g?)r   rh   r   r\   r   	who_wantsidr	  r   r   r   r,   rt   r   r	   )r_   rR   rS   rT   r   rh   r   rd   rd   re   $test_garbage_collection_with_scatter  s    r  i  )rU   rL   c       	      c   s   |  td}|V }|j}~dd l}|  tjV  | j| dksDtx:||j	kr\|j	| j
sp||jksp||jkr~tdV  qFW |  td}|j| jkst|V }||kstd S )Nr   r   g?)rZ   r<   rh   r   r   r   r   r	  r\   rt   r   datar	   r   )	r_   rR   rS   rT   r`   result1Zxkeyr   result2rd   rd   re   test_recompute_released_key  s    ,r  c             c   s"   ddl m} | |d}|V  d S )Nr   )r	   rl   )r,   r	   rZ   )r_   rR   rS   rT   r	   r`   rd   rd   re   $test_long_tasks_dont_trigger_timeout  s    r  c       	      c   s   d|_ d|_ | td}| t|}| t|}t|||gV  |j|jkrb|j|j= ||j |j|jkr|j|j= ||j tjV  | t	||}|V }|dkst
d S )NFrk   r   )validaterZ   r<   r#   rh   r  release_keyr   r   r   r\   )	r_   rR   rS   rT   r`   rb   rc   wra   rd   rd   re   test_missing_data_heals  s     

r  c             c   s   d|_ d|_ | ttd\}}}t|||gV  xL||gD ]@}x:||gD ].}|j|jkrJ|j|j= tjV  |	|j qJW q<W | 
|||gV \}	}
}|	|
|fdkstd S )NFrl   )rk   r   rl   )r  rp   r<   rq   r#   rh   r  r   r   r  rv   r\   )r_   rR   rS   rT   r`   rb   rc   rn   r  r   yyzzrd   rd   re   "test_gather_robust_to_missing_data  s    
r  c             c   s   d|_ d|_ | td}| t|}| t|}| t|}t|gV  xL||gD ]@}x:||gD ].}	|	j|jkr`|j|	j= tjV  ||	j q`W qRW | 	|gV }
|
ttttdgkst
d S )NFrk   )r  rZ   r<   r#   rh   r  r   r   r  rv   r\   )r_   rR   rS   rT   r  r`   rb   rc   workerZdatumra   rd   rd   re   )test_gather_robust_to_nested_missing_data  s    
r  c             C   sl   |  td}|  td}t|}t|t|ks4tt|t|ksHt| j|j   |t|kshtd S )Nrk   )rZ   r<   r%   r\   r   rh   finish)r_   rR   rS   rT   r`   rb   tokrd   rd   re   test_tokenize_on_futures  s    r  Zlinuxz Need 127.0.0.2 to mean localhost)reason)z	127.0.0.2r   c             c   s   | j td|jhd}| j t||jhd}t||gV  |j|j |jhksNt|j|jks^t|j|j |jhksvt|j|jkstd S )Nrk   )r   )rZ   r<   r   r#   host_restrictionsrh   r\   r  )r_   rR   rS   rT   r`   rb   rd   rd   re   test_restrictions_submit   s    r!  c             c   s   | j td|jhd}| j t||jhd}t||gV  |j|j |jhksNt|j|jks^t|j|j |jhksvt|j|jkstd S )Nrk   )r   )rZ   r<   r   r#   worker_restrictionsrh   r\   r  )r_   rR   rS   rT   r`   rb   rd   rd   re   test_restrictions_ip_port/  s    r#  c          	   c   s2  | j ttd|jhd}t|V  t|jdd |D ks>t|jrHtx$|D ]}|j|j	 |jhksNtqNW | j tdddg|jh|j|jh|jhgd}t|V  |j|d j	 |jhkst|j|d	 j	 |j|jhkst|j|d
 j	 |jhkstt
t" | j tdddg|jhgd W d Q R X d S )Nrg   )r   c             S   s   h | ]
}|j qS rd   )rh   )ri   r`   rd   rd   re   r   C  s    z(test_restrictions_map.<locals>.<setcomp>rV   rW   r   r   rk   r   )rp   r<   rq   r   r#   r  r  r\   r   rh   r   r   
ValueError)r_   rR   rS   rT   r   r`   rd   rd   re   test_restrictions_map<  s    




 r%  c             c   sr   dt dft dfd}|jh|jhd}| j|ddg|dd}|V }|d	d
gksRtd|jks`td|jksntd S )Nrk   r`   rb   )r`   rb   rc   )rb   rc   rc   F)r2   r   rl   )r<   r   r   r\   r  )r_   rR   rS   rT   dskZrestrictionsr   ra   rd   rd   re   test_restrictions_getU  s    r'  c          
   c   sn   | j tddhd}y|V  ds"tW nD tk
rh } z&dt|ksFt|jt|ksXtW d d }~X Y nX d S )Nr   zbad-address)r   F)rZ   r<   r\   r$  r   rh   )r_   rR   rS   rT   rc   r  rd   rd   re   *dont_test_bad_restrictions_raise_exceptionc  s    r(  c             c   s^   |  ttd}t|V  | V  |j|jks4t| |V }|t	t ttdksZtd S )NrX   )
rp   r<   rq   r#   r   r   r   r\   rv   ry   )r_   rR   rS   rT   r   ra   rd   rd   re   test_remove_workern  s    

r)  )r   rL   c             c   s   |  td|  td|  td|  tdg}t }xD|d j|d j  krTdksvn t |d k shttdV  q4W | |d |d gV }|ddgkstd S )Nrk   r   r   r  rg   g{Gz?rl   )	rZ   r<   r?   r,   r   r\   r   r	   rv   )r_   rR   r  r   r   ra   rd   rd   re   test_errors_dont_block{  s    


$r*  c       	      c   s   g fdd}|  |dddg}|V }|s,t| j |dddgdddgd	}|V }|sVt|  td}|  td}|  |||g}|V }|std S )
Nc             S   s   t | tot |tS )N)r]   ry   )r`   rc   rd   rd   re   assert_list  s    z'test_submit_quotes.<locals>.assert_listrk   r   rl   rm   rg   r   )rc   )rZ   r\   r<   )	r_   rR   rS   rT   r+  r`   ra   rb   rc   rd   rd   re   test_submit_quotes  s    r,  c             c   s   g fdd}|  |dddgdgg}| |V }t|s<t| j |dddgdggdgd}| |V }t|srt|  |dddgdggg gd }| |V }t|std S )	Nc             S   s   t | tot |tS )N)r]   ry   )r`   rc   rd   rd   re   r+    s    z$test_map_quotes.<locals>.assert_listrk   r   rl   rm   rV   )rc   )rp   rv   rs   r\   )r_   rR   rS   rT   r+  r   ra   rd   rd   re   test_map_quotes  s     r-  c       	      c   s   t | j| jfddV }|jtjdddd}|V }t | j| jfddV }|jtjdddd}|V }||kslt| V  | V  d S )NT)r   r   i  )r   )r!   r   r   rZ   randomrandintr\   r   )	rR   rS   rT   r_   r`   r   rn   rb   r  rd   rd   re   *test_two_consecutive_clients_share_results  s    
r0  c             c   s:   |  td}dt|fi}| j|dddV }|dks6td S )Nrk   rb   F)r2   rl   )rZ   r8   r<   r   r\   )r_   rR   rS   rT   r`   r&  ra   rd   rd   re    test_submit_then_get_with_Future  s    r1  c             c   s6   |  td}d|i}| j|dddV }|dks2td S )Nrk   rb   F)r2   r   )rZ   r<   r   r\   )r_   rR   rS   rT   r`   r&  ra   rd   rd   re   test_aliases  s    r2  c             c   s   t dfddtddfdddgfddddgfdddt dfddgfg}xF|D ]>\}}| j||dd	V }t|tt||ksttjV  qLW d S )
Nrk   r`   rb   rc   )r`   rb   rc   r  r  )r`   rb   F)r2   )r<   r   r   ry   r   r\   r   r   )r_   rR   rS   rT   Zdsk_keysr&  r   ra   rd   rd   re   test_aliases_2  s    r3  c             c   s  |  ddiV }t|d ts"t|jddksF|jddksFt|jdgdd }|j|ksp|j|kspt|jdddt	dikst| 
|d gV }|dgkst|  dgV \}t|tst|j|jdks|j|jdkst| 
|gV }|j|jgd|j }	|j|j js*t|j|j |j|j jksf|j|j |j|j jksft|jdddt	d|jt	dikst|dgkst| t||d }
|
V }|dkst| 
|
|gV }|ddgkstd S )Nrb   rX   )r   F)summaryrV   r   )r   r]   r"   r\   r  r   Zget_who_hasr   
get_nbytesr/   rv   rh   rt   r   r   rZ   r   )r_   rR   rS   rT   dZ	y_who_hasr  r`   r   Z	x_who_hasrc   ra   rd   rd   re   test__scatter  s0    $
( (r7  c             c   s   |  ddiV }t|tstt|dgks0txTdgddhtdggD ]:}|  |V }t|t|sjtt|dkszt|  qHW |  t	dV }t|tstt|dkst|  d S )Nr`   rk   )rk   rg   )
r   r]   dictr\   ry   	frozensettyperr   r^   rq   )r_   rR   rS   rT   r6  seqr   rd   rd   re   test__scatter_types  s    r<  c             c   s0   |  dV }t|tst|V }|dks,td S )Nrk   )r   r]   r"   r\   )r_   rR   rS   rT   r`   ra   rd   rd   re   test__scatter_non_list  s    r=  c             c   s<   |  dgV \}|  dgV \}|j|jks0t|  d S )Nrk   )r   rh   r\   r^   )r_   rR   rS   rT   rd   rd   re   test_scatter_hash  s    r>  c       	      #   s`   ddl m} G dd dt}g  || fdd}| }| |V } rX d |ks\td S )Nr   )normalize_tokenc               @   s   e Zd ZdS )z*test_scatter_tokenize_local.<locals>.MyObjN)__name__
__module____qualname__rd   rd   rd   re   MyObj&  s   rC  c                s     |  dS )Nr`   )r   )r`   )r   rd   re   rn   +  s    
z&test_scatter_tokenize_local.<locals>.f)Z	dask.baser?  objectregisterr   r\   )	r_   rR   rS   rT   r?  rC  rn   objr   rd   )r   re   test_scatter_tokenize_local"  s    rG  c       	      c   sh   t d}t d}xNd|d|ddddgigD ]*}| |V }|V }t|t|ks6tq6W d S )Nnumpypandasrk   rg   r`   r   rl   )r   r   r   	DataFramer   r   r\   )	r_   rR   rS   rT   nppdr`   r   ra   rd   rd   re   test_scatter_singletons6  s    

&rM  c             c   s    |  dV }|jdstd S )N{   int)r   rh   r   r\   )r_   rR   rS   rT   r   rd   rd   re   test_scatter_typename@  s    rP  c             c   sL   |  dV }|  dV }|j|jks(t| j dddV }|j|jksHtd S )NrN  F)hash)r   rh   r\   )r_   rR   rS   rT   r`   rb   rc   rd   rd   re   r>  F  s
    c             c   sf   | j dtdfidgddV \}dd l}|  t }x,| jd r`tdV  t |d k s6tq6W d S )Nr`   rk   F)r2   r   g{Gz?r   )	r   r<   r   r   r,   r	  r   r	   r\   )r_   rR   rS   rT   r`   r   r   rd   rd   re   test_get_releases_dataP  s    rR  c          	   C   sr   t | d }t  |kstW d Q R X tt t   W d Q R X t | d }t  |ksdtW d Q R X d S )Nr   )r!   currentr\   r   r   r$  )rR   rS   rT   r_   rd   rd   re   test_Current\  s    rT  c             C   s   t  d ksttt t  W d Q R X t \}\}}t|d | dz}t  |ksZtt |kshtt|d | dB}t  |kstt |kstt||kstt||kstW d Q R X W d Q R X W d Q R X t  d kstd S )Nr   )rO   )r&   r\   r   r   r$  r'   r6   r!   )rO   rR   rS   rT   r_   rn   rd   rd   re   test_global_clientse  s    .rU  c          	   c   sb   |  dd }|  t|}tt |V  W d Q R X |  t|}tt |V  W d Q R X d S )Nc               S   s   dd S )Nrk   r   rd   rd   rd   rd   re   r   x  s    z-test_exception_on_exception.<locals>.<lambda>)rZ   r<   r   r   r   )r_   rR   rS   rT   r`   rb   rc   rd   rd   re   test_exception_on_exceptionv  s    rV  c             c   sl   |  dgV \}|jdd|jtdiks.t| t|}|V  |jdd|jtd|jtdikshtd S )Nrk   F)r4  r   )r   r5  rh   r/   r\   rZ   r<   )r_   rR   rS   rT   r`   rb   rd   rd   re   test_get_nbytes  s    rW  c             c   s|   | j td|jgd}| j tttd|jgd}| ||gV  |  dd ||}|V  |j|j j|j	|j
 hksxtd S )Nrk   )r   r   c             S   s   d S )Nrd   )r`   rb   rd   rd   re   r     s    z/test_nbytes_determines_worker.<locals>.<lambda>)rZ   r
   r   tuplerq   rv   rt   rh   r   r   r   r\   )r_   rR   rS   rT   r`   rb   rc   rd   rd   re   test_nbytes_determines_worker  s    rY  c          	   c   s   t tdddd}t tdddd}t tdd||}| |}tt |V  W d Q R X |  tdd |j	
 D rtd S )NT)r   rk   r   r   c             s   s   | ]}|j V  qd S )N)r   )ri   tsrd   rd   re   rj     s    z7test_if_intermediates_clear_on_error.<locals>.<genexpr>)r   r>   r   r   r   r   r   r^   anyrt   valuesr\   )r_   rR   rS   rT   r`   rb   rc   rn   rd   rd   re   $test_if_intermediates_clear_on_error  s    
r]  c       
      #   s   t d}| j|jdgd dd}| |j|}| t|}dd }d jd	< | |||gd }	t|gV  t|	V  t fd
dt||	D dkst	d S )NrH  i'  rV   F)r   c             S   s   d S )Nrd   )r`   rb   rd   rd   re   rn     s    z7test_pragmatic_move_small_data_to_large_data.<locals>.fgMbP?rn   c             3   s0   | ](\}} j |j j j |j jV  qd S )N)rt   rh   r   issubset)ri   lr)rR   rd   re   rj     s   z?test_pragmatic_move_small_data_to_large_data.<locals>.<genexpr>r   )
r   r   rp   r   ru   rZ   task_durationr#   zipr\   )
r_   rR   rS   rT   rK  ZlistsZsumsr|   rn   r   rd   )rR   re   ,test_pragmatic_move_small_data_to_large_data  s    


rc  c             c   sR   t dft dfd}| j|dddV }| j|dddV }|dksBt|dksNtd S )	Nrk   r   ))r`   r   rg   )r`   r   F)r2   rg   rl   )r<   r   r\   )r_   rR   rS   rT   r&  r`   rb   rd   rd   re   test_get_with_non_list_key  s
    rd  c          	   c   s@   t ddftdfd}tt | j|dddV  W d Q R X d S )Nrk   r   r`   )r`   rb   rb   F)r2   )r>   r<   r   r   r   r   )r_   rR   rS   rT   r&  rd   rd   re   test_get_with_error  s    re  c          	   C   s:   t ddftdfd}tt | |d W d Q R X d S )Nrk   r   r`   )r`   rb   rb   )r>   r<   r   r   r   r   )r_   r&  rd   rd   re   test_get_with_error_sync  s    rf  c             c   sd   | j dddg|jgdV  t|jdks,t|jr6t| j ddg|jgdV  t|jdks`td S )Nrk   r   rl   )r   rm   rg   )r   r   rr   r  r\   name)r_   rR   rS   rT   rd   rd   re   test_directed_scatter  s
    
rh  c             C   s`   | j dddg|d gd}t|| jj}t||d  t|ksDtt||d  dks\td S )Nrk   r   rl   r   )r   r   )r   r2   	schedulerhas_whatrr   r\   )r_   rR   rS   rT   rO   r   rj  rd   rd   re   test_directed_scatter_sync  s    rk  c          
   C   s   |  dddg}dddg| |ks(tdd tdD }|  |}t|tsRtt|}| |dksltt|}t|dkst| |dddd	d
ddddg	kstd S )Nrk   r   rl   c             s   s   | ]
}|V  qd S )Nrd   )ri   ird   rd   re   rj     s    z(test_iterator_scatter.<locals>.<genexpr>rV   r   r   rm   rg   r   r   r   )	r   rv   r\   rq   r]   r+   nextry   rr   )r_   aar   r   rS   rd   rd   re   test_iterator_scatter  s    
ro  c             C   sd   ddl m} | }xtdD ]}|| qW | |}t||sFt| }| |dks`td S )Nr   )QueuerV   )	distributed.compatibilityrp  rq   putr   r]   r\   r   rv   )r_   rp  qr6  r   rS   rd   rd   re   test_queue_scatter  s    
rt  c             C   s   ddl m} |dd}| j|dd}|jdks2t| |}|jrFt| }| |}|jr`t| j|dd}|jdks|t|dd}| |}|jrtd S )Nr   )rp  rl   )maxsizerV   )rq  rp  r   ru  r\   rv   )r_   rp  rs  outlocalrd   rd   re   !test_queue_scatter_gather_maxsize  s    






rx  c       	      C   s   ddl m} | }ttd}x|D ]}|| q$W | |}t||sNt| |}t||sftg }xtdD ]}|	|
  qtW ||kstd S )Nr   )rp  rV   )rq  rp  ry   rq   rr  r   r]   r\   rv   r   r   )	r_   rp  rs  Zqinr6  r   ffZqoutrn   rd   rd   re   test_queue_gather  s    


rz  zintermittent blocking failuresc             C   s  t td}dd |D }| |}t|ts2t| |}t|tsJtt |}||ks^tdddtdtddg}d	d |D }| |}| |}t |}|d d
 |d d
 kstt|d
 tst|d
 j|d
 jkst|dd  |dd  kstd S )NrV   c             s   s   | ]
}|V  qd S )Nrd   )ri   r6  rd   rd   re   rj   ,  s    z'test_iterator_gather.<locals>.<genexpr>rS   rT   r_   rn   r6  c             s   s   | ]
}|V  qd S )Nrd   )ri   r6  rd   rd   re   rj   8  s    rl   rm   )	ry   rq   r   r]   r+   r\   rv   StopIterationr   )r_   c2Zi_inr   r   ry  Zi_outrd   rd   re   test_iterator_gather(  s"    



r}  c             c   s   | j dddV }|j|jks,|j|jks,t|j|j js>t|jdksLt|V }|dks^t|jd jd d rvt|V }|jd jd d rt| 	|V }|jd jd d rtd S )	NrN  T)directr  opr   r   rv   )
r   rh   r  r\   rt   r   r   counters
componentsrv   )r_   rR   rS   rT   r   ra   rd   rd   re   test_scatter_directD  s    r  c             c   sV   t d}|d}| j|ddV }|V }|||s:t|jd jd d rRtd S )NrH  rg   T)r~  r  r   r   )r   r   r   r   Zallcloser\   r  r  )r_   rR   rS   rT   rK  r`   r   ra   rd   rd   re   test_scatter_direct_numpyU  s    

r  c             c   s   | j ddddV }|j|jks"t|j|jks2t|j|j j|j|j |j|j hks\t|V }|dksnt|jd j	d d rtd S )Ni  T)r~  	broadcastr  r   r   )
r   rh   r  r\   rt   r   r   r   r  r  )r_   rR   rS   rT   future2ra   rd   rd   re   test_scatter_direct_broadcast_  s    r  rm   c             g   s<   | j dddgddV }tdd |D ddddgks8td S )	Nrk   r   rl   T)r~  c             S   s   g | ]}t |jqS rd   )rr   r  )ri   r  rd   rd   re   r   n  s    z0test_scatter_direct_balanced.<locals>.<listcomp>r   )r   sortedr\   )r_   rR   r   r   rd   rd   re   test_scatter_direct_balancedk  s    r  c             '   s   | j ddgdd jdV }|d jd jks4t|d jd jksLt| j ddgdddd d d	 D d
V } fdd|D std S )NrN  i  Tr   )r~  r   rk   c             S   s   g | ]
}|j qS rd   )r   )ri   r  rd   rd   re   r   y  s    z8test_scatter_direct_broadcast_target.<locals>.<listcomp>rl   )r~  r  r   c             3   s@   | ]8}d d D ]&}|j |jko4|j j|j  jkV  qqd S )Nrl   )rh   r  r   rt   r   )ri   rn   r  )rR   r   rd   re   rj   z  s   z7test_scatter_direct_broadcast_target.<locals>.<genexpr>)r   r   rh   r  r\   )r_   rR   r   r   rd   )rR   r   re   $test_scatter_direct_broadcast_targetq  s    r  c          	   c   s2   t ttjf | jddddV  W d Q R X d S )NrN  Tg?)r~  rU   )r   r   r$  r   r   r   )r_   rR   rd   rd   re   test_scatter_direct_empty  s    r  rg   )rL   rU   r   c             g   sJ   g }x*t dD ]}| j|ddV }|| qW tdd |D sFtd S )NrV   T)r~  c             s   s   | ]}|j V  qd S )N)r  )ri   r  rd   rd   re   rj     s    z4test_scatter_direct_spread_evenly.<locals>.<genexpr>)rq   r   r   rs   r\   )r_   rR   r   r   rl  r   rd   rd   re   !test_scatter_direct_spread_evenly  s
    r  r~  Fr  c             C   sN   | j dddg||d}| j||d}|dddgks6tttdj|d d S )Nrk   r   rl   )r~  r  )r~  )r   rv   r\   r   r<   r   )r_   r~  r  r   r   rd   rd   re   test_scatter_gather_sync  s    r  c             c   s8   |  dddgV }| j|ddV }|dddgks4td S )Nrk   r   rl   T)r~  )r   rv   r\   )r_   rR   rS   rT   r   r  rd   rd   re   test_gather_direct  s    r  c             #   s4    fddt dD }t|V  |jr,|js0td S )Nc                s   g | ]}  t|qS rd   )rZ   r<   )ri   rl  )r_   rd   re   r     s    z3test_many_submits_spread_evenly.<locals>.<listcomp>rV   )rq   r#   r  r\   )r_   rR   rS   rT   r   rd   )r_   re   test_many_submits_spread_evenly  s    
r  c             c   sL   |  tdd}| V }tjd dkrHtdd tdt|D sHtd S )Nrk   r   rl   c             s   s   | ]}d |kV  qdS )zx / yNrd   )ri   linerd   rd   re   rj     s   z!test_traceback.<locals>.<genexpr>)	rZ   r>   	tracebackrw   rx   r[  r   
extract_tbr\   )r_   rR   rS   rT   r`   r   rd   rd   re   test_traceback  s
    
r  c             c   sh   y | j dtddfidddV  W nB tk
rb   t \}}}t|}tdd |D s^tY nX d S )Nr`   rk   r   F)r2   c             s   s   | ]}d |kV  qdS )zx / yNrd   )ri   r  rd   rd   re   rj     s    z%test_get_traceback.<locals>.<genexpr>)	r   r>   r   rw   r   r  	format_tbr[  r\   )r_   rR   rS   rT   exc_type	exc_valueexc_tracebackr   rd   rd   re   test_get_traceback  s     
r  c       	      c   sf   |  tdd}y| |V  W nB tk
r`   t \}}}t|}tdd |D s\t	Y nX d S )Nrk   r   c             s   s   | ]}d |kV  qdS )zx / yNrd   )ri   r  rd   rd   re   rj     s    z(test_gather_traceback.<locals>.<genexpr>)
rZ   r>   rv   r   rw   r   r  r  r[  r\   )	r_   rR   rS   rT   r`   r  r  r  r   rd   rd   re   test_gather_traceback  s    
r  c             C   s   |  tdd}| }tjd dkrDtdd tt|D sDt|  t	|}| }t
tdt|t
tdt|st|  tdd}| }|d kstd S )Nrk   r   rl   c             s   s    | ]}t |trd |kV  qdS )zx / yN)r]   r   )ri   r  rd   rd   re   rj     s   z&test_traceback_sync.<locals>.<genexpr>r   )rZ   r>   r  rw   rx   r[  r   r  r\   r<   r  r   
issuperset)r_   r`   r   rb   Ztb2rc   rd   rd   re   test_traceback_sync  s    r  c       	   
   c   s|   dd }zZxTdD ]L}t dd|}| |V  W d Q R X | j|dd}|V }||kstqW W d dtjkrvtjd= X d S )	Nc              S   s   dd l } |  S )Nr   )myfilern   )r  rd   rd   re   r     s    ztest_upload_file.<locals>.g)rN  i  z	myfile.pyzdef f():
    return {}F)r   r  )r3   formatupload_filerZ   r\   rw   modules)	r_   rR   rS   rT   r   valuefnr`   ra   rd   rd   re   test_upload_file  s    

r  c          	   c   s&   t dd}| |V  W d Q R X d S )Nr   )r3   r  )r_   rR   rS   rT   r  rd   rd   re   test_upload_file_no_extension  s    r  c             c   s   dd }zxdD ]z}t dd|`}tdd}|j|tj|d W d Q R X | dV  | j	|d	d
}|V }	|	|kst
W d Q R X qW W d tjdrtd dtjkrtjd= x,tjD ]"}
tj|
dkrtj|
 P qW X d S )Nc              S   s   dd l } |  S )Nr   )r  rn   )r  rd   rd   re   r     s    ztest_upload_file_zip.<locals>.g)rN  i  z	myfile.pyzdef f():
    return {}z
myfile.zipr  )ZarcnameF)r   r  )r3   r  zipfileZZipFilewriteospathbasenamer  rZ   r\   existsremoverw   r  )r_   rR   rS   rT   r   r  Z
fn_my_filerc   r`   ra   r  rd   rd   re   test_upload_file_zip  s$    
 

r  c             c   s
  |j s
t|j sttdd}tdd}| j|ddV  | |V  x||gD ]}tjtj|j dsrttjtj|j dstttj|j d}|	 dkstW d Q R X ttj|j d}|	 dkstW d Q R X qRW W d Q R X W d Q R X d S )Nr  abcZmyfile2defr`   )Zremote_filename)
Z	local_dirr\   r3   Z_upload_large_filer  r  r  joinopenread)r_   rR   rS   rT   r  Zfn2r  rn   rd   rd   re   test_upload_large_file  s    

r  c          	   C   sF   dd }t dd*}| | | |}| dks8tW d Q R X d S )Nc              S   s   dd l } | jS )Nr   )r  r`   )r  rd   rd   re   r   $  s    z test_upload_file_sync.<locals>.gz	myfile.pyzx = 123rN  )r3   r  rZ   ra   r\   )r_   r   r  r`   rd   rd   re   test_upload_file_sync#  s
    

r  c          
   c   s<   t dd(}tt | |V  W d Q R X W d Q R X d S )Nz	myfile.pyzsyntax-error!)r3   r   r   SyntaxErrorr  )r_   rR   rS   rT   r  rd   rd   re   test_upload_file_exception.  s    r  c          
   C   s:   t dd&}tt | | W d Q R X W d Q R X d S )Nz	myfile.pyzsyntax-error!)r3   r   r   r  r  )r_   r  rd   rd   re   test_upload_file_exception_sync5  s    r  c       	      c   s   t | j| jfddV }t | j| jfddV }|td}|td}|j|ksRt|j|ks`t|V }|V }|dksxt|dkst|t||}|j|kst|V }|dkst| V  | V  d S )NT)r   rk   r   rl   rg   )	r!   r   r   rZ   r<   rL   r\   r   r   )	rR   rS   rT   r`   rb   r   r  rc   r  rd   rd   re   test_multiple_clients;  s     
r  c             c   s   ddl m} |d}|t|}|t|}| j||dgdd\}}	}
t|tsTtt|	tsbt|
dksnt| ||	gV }|ddgkstt| |tstt| |gt	t
fstd S )Nr   )r   rk   rl   F)r2   r   )dask.delayedr   r<   r=   r   r]   r"   r\   rv   rX  ry   )r_   rR   rS   rT   r   r`   rb   rc   r  r  rn  ra   rd   rd   re   test_async_computeR  s    r  c             c   sx   |  dddV }|d |d  }}ddlm} |t|t||t|}| |}	| |	gV \}
|
dksttd S )	Nrk   r   ))r`   rk   )rb   rk   )r`   rk   )rb   rk   r   )r   rg   )r   r  r   r   r<   r   rv   r\   )r_   rR   rS   rT   r6  r`   rb   r   rc   r  ra   rd   rd   re   test_async_compute_with_scattere  s    
r  c             C   sJ   t d}t t|}t t|}| j||gdd\}}||fdksFtd S )Nrk   T)r2   )r   r   )r   r<   r=   r   r\   )r_   r`   rb   rc   r  r  rd   rd   re   test_sync_computer  s
    r  c       
      c   s   |  dddgV \}}}|j|jks4|j|jks4t|j|jksP|j|jksPt|j|jksl|j|jkslt| |||gV \}}}	|||	fdkstd S )Nrk   r   rl   )rk   r   rl   )r   rh   r  r\   rv   )
r_   rR   rS   rT   r`   rb   rc   r   r  r  rd   rd   re   test_remote_scatter_gather{  s    r  c             c   s6   |  dd d}|  dd |}|V }|dks2td S )Nc             S   s   | d S )Nrk   rd   )r`   rd   rd   re   r     s    z.test_remote_submit_on_Future.<locals>.<lambda>rk   c             S   s   | d S )Nrk   rd   )r`   rd   rd   re   r     s    rl   )rZ   r\   )r_   rR   rS   rT   r`   rb   ra   rd   rd   re   test_remote_submit_on_Future  s    r  c             C   s8   |    |    |    | td}| dks4td S )Nrk   r   )r   rZ   r<   ra   r\   )r_   r`   rd   rd   re   test_start_is_idempotent  s
    r  c             c   s   |j |j|j |j|j ikst| td}| td}| t||}|V }|dksVt|V }|dksht| dddgV \}}	}
| ||	|gV \}}}|||fdkst| jtdftddfdd	d
dV }|dkstd S )Nrk   r   rg   rl   )rk   r   r   r`   rV   )r`   rb   rb   F)r2   r   )	r   r   r\   rZ   r<   r   r   rv   r   )r_   rR   rS   rT   r`   rb   rc   ra   ABCZAAZBBr   rd   rd   re   test_client_with_scheduler  s    "r  c          	   #   s*  j |j  j |j }| jtd|jd}|V  j|j j hksHtj	rRt| jtd|jdd}|V  j|j j hkst|jj	kst| j
ttdd|jdd}t|V  t fdd	|D std
d |D j	st| jtdddd}|V  j|j jst|jj	ks,t| j
ttddddd}t|V  tfdd	|D sjtdd |D j	sttt | jtddd W d Q R X tt | j
tdgdd W d Q R X tt | jtdddd W d Q R X tt | j
tdgddd W d Q R X d S )Nrk   )r   r   T)r   allow_other_workersrl      c             3   s"   | ]}j |j j hkV  qd S )N)rt   rh   r   )ri   rn   )awsrR   rd   re   rj     s    z*test_allow_restrictions.<locals>.<genexpr>c             S   s   h | ]
}|j qS rd   )rh   )ri   rn   rd   rd   re   r     s    z*test_allow_restrictions.<locals>.<setcomp>   z	127.0.0.3ro   c             3   s   | ]} j |j jV  qd S )N)rt   rh   r   )ri   rn   )rR   rd   re   rj     s    c             S   s   h | ]
}|j qS rd   )rh   )ri   rn   rd   rd   re   r     s    )r  rX   z	127.0.0.1zHello!)r   r   rZ   r<   r   rt   rh   r   r\   loose_restrictionsrp   rq   r#   rs   r^  r   r   r$  r   )r_   rR   rS   rT   bwsr`   r   rd   )r  rR   re   test_allow_restrictions  s<    


r  TrueZbecausec           
   C   s   yt ddd W n< ttjfk
rL }  zdt|  ks<tW d d } ~ X Y nX yt ddd W n< ttjfk
r }  zdt|  kstW d d } ~ X Y nX d S )Nz123.123.123.123:1234g?)rU   Zconnectz127.0.0.1:1234)r!   IOErrorr   r   r   lowerr\   )r  rd   rd   re   test_bad_address  s    &r  c          
   c   s   dd }|  |d}y
|V  W n6 tk
rT } ztt|dk sDtW d d }~X Y nX | V }tdd tt|D std S )Nc             S   s   t dd d S )NrS   i )r$  )r`   rd   rd   re   bad  s    ztest_long_error.<locals>.badrV   i c             s   s$   | ]}t |trt|d k V  qdS )i N)r]   r   rr   )ri   r  rd   rd   re   rj     s   z"test_long_error.<locals>.<genexpr>)	rZ   r$  rr   r   r\   r  rs   r   r  )r_   rR   rS   rT   r  r`   r  r   rd   rd   re   test_long_error  s    
&
r  c             c   s   ddd}|  ttd}| j ||dd}| |V }|dd tdD ksPt| td}| j||d	d}	|	V }
|
d
ks~td S )NrV   c             S   s   | | S )Nrd   )r`   rb   rd   rd   re   rn     s    z*test_map_on_futures_with_kwargs.<locals>.frX   )rb   c             S   s   g | ]}|d  d qS )rk   rX   rd   )ri   rl  rd   rd   re   r     s    z3test_map_on_futures_with_kwargs.<locals>.<listcomp>r      i-  )rV   )rp   r<   rq   rv   r\   rZ   )r_   rR   rS   rT   rn   r   r   r   r   r  ra   rd   rd   re   test_map_on_futures_with_kwargs  s    
r  c               @   s   e Zd Zdd Zdd ZdS )BadlySerializedObjectc             C   s   dS )Nrk   rd   )selfrd   rd   re   __getstate__
  s    z"BadlySerializedObject.__getstate__c             C   s   t dd S )Nzhello!)r   )r  staterd   rd   re   __setstate__  s    z"BadlySerializedObject.__setstate__N)r@  rA  rB  r  r  rd   rd   rd   re   r  	  s   r  c               @   s   e Zd Zdd Zdd ZdS )FatallySerializedObjectc             C   s   dS )Nrk   rd   )r  rd   rd   re   r    s    z$FatallySerializedObject.__getstate__c             C   s   t d dd l}|d d S )Nz1This should never have been deserialized, closingr   )printrw   exit)r  r  rw   rd   rd   re   r    s    z$FatallySerializedObject.__setstate__N)r@  rA  rB  r  r  rd   rd   rd   re   r    s   r  c             c   s^   t  }| t|}| ttd}| |V }t|ttttdksLt|jdksZtd S )NrV   error)	r  rZ   r<   rp   rq   rv   ry   r\   r   )r_   rR   rS   rT   or   r   r   rd   rd   re   test_badly_serialized_input  s    r  r  c             C   s`   t  }|t|}t }x4td |  \}}d|kr8P t | dk stqW |jdks\td S )Ng{Gz?zhello!rX   r  )r  rZ   r<   r,   r	   
readouterrr\   r   )capsysr_   r  r   r   rv  errrd   rd   re   "test_badly_serialized_input_stderr'  s    r  c          
   C   s   t tdd g}tdd\}\}}}t|d | d8}x0|D ](}||}|jj|ksXtd|ks<tq<W W d Q R X x |D ]}||}d|ksxtqxW W d Q R X d S )	Nc             S   s   |   S )N)r   )r`   rd   rd   re   r   7  s    ztest_repr.<locals>.<lambda>rl   )nworkersr   )rO   3znot connected)r   r   r6   r!   ri  r   r\   )rO   ZfuncsrR   rS   rT   r_   r   textrd   rd   re   	test_repr6  s    

r  c             C   s   |    d S )N)r   )r_   rR   rS   rT   rd   rd   re   test_repr_asyncD  s    r  c              c   sV   t dd ddV } t| ddV }z| }| jj|ks8tW d | V  |  V  X d S )NFT)	processesdiagnostics_portr   )r   )r   r!   r   ri  r   r\   r   )r6   rL   r  rd   rd   re   test_repr_localclusterI  s    

r  c             c   s  | j tddd}|  td}| j t|||jgdd}t|||gV  |j|jrVt|j|jrhtt	|j
|j|j|jhkst|j|jg| jd |j|j
kst|j|jg| jd |j|j
kst|j|j
kst|j
|j jrt|j|jg| jd |j
rtd S )Nrk   r   )r   T)r   r  )r   rL   )rZ   r<   r   r   r#   waiting_datar   rh   r\   r  rt   client_releases_keysr  Z
dependents)r_   rR   rS   rT   r`   rb   rc   rd   rd   re   test_forget_simpleV  s    r  c          
   c   s  |  ttdV \}}}}| t||}| t||}	| t||}
| t|
|}t||||||
|	|gV  t|jdd ||
|	|||||gD kst|j	|j
g| jd t|jdd ||
|	|||||gD kst|j	|j
g| jd t|jdd |
|	||||gD kst|j	|j
g| jd t|jdd |
|	|||gD ksTt|j
|jksftt }xB|j
|jks|j
|jkrtdV  t |d	 k sntqnW |j	|
j
g| jd t|jd
d |	|||gD kstd S )Nrm   c             S   s   h | ]
}|j qS rd   )rh   )ri   rn   rd   rd   re   r   x  s    z&test_forget_complex.<locals>.<setcomp>)r   rL   c             S   s   h | ]
}|j qS rd   )rh   )ri   rn   rd   rd   re   r   {  s    c             S   s   h | ]
}|j qS rd   )rh   )ri   rn   rd   rd   re   r   ~  s    c             S   s   h | ]
}|j qS rd   )rh   )ri   rn   rd   rd   re   r     s    g{Gz?rV   c             S   s   h | ]
}|j qS rd   )rh   )ri   rn   rd   rd   re   r     s    )r   ry   rq   rZ   r   r#   r  rt   r\   r  rh   r  r,   r  r   r	   )r  rR   r  r  rS   rT   r_   r6  abcdacacabr   rd   rd   re   test_forget_complexn  s(    ,,*(r  c             #   s   t tdd  fddtdD \}}}} t||dd} t||dd}	 t||d	d}
 t|
|d
d}| |
|g\}}|  x$tdD ]}tdV  |  qW |j|j	g| j
d |  x&|j	|j	|j	gD ]}||jkstqW d S )NT)r   c                s   g | ]} t |qS rd   )r8   )ri   rl  )delayed2rd   re   r     s    z)test_forget_in_flight.<locals>.<listcomp>rm   r  )dask_key_namer  r  r  rg   g{Gz?)r   rL   )r   r   rq   r9   r   r^   r   r	   r  rh   r  rt   r\   )r  rR   r  r  rS   rT   r_   r6  r  r  r  r  r`   rb   rl  r   rd   )r  re   test_forget_in_flight  s    r  c             c   s~  |  tdd}|  t|}|  t|}t|gV  |j|jksBt|j|jksRt|j|jksbt|j|jksrt|j|jg| j	d |j|jkst|j|jkst|j|jkst|j|jkst|j|jg| j	d |j|jkst|j|jkst|j|jkst|j|jkst|j|jg| j	d |j|jksDt|j|jksVt|j|jksht|j|jksztd S )Nrk   r   )r   rL   )
rZ   r>   r<   r#   rh   
exceptionsr\   Zexceptions_blamer  r  )r_   rR   rS   rT   r`   rb   rc   rd   rd   re   test_forget_errors  s.    r  c             C   sP   t | }t| }| jj|ks t| jj|ks0tt d|ks@td|ksLtd S )Nr   Zcores)r   r   ri  r   r\   )r_   rR   r`  rd   rd   re   test_repr_sync  s    r  c             c   sh   |  td}|  td}| j t|||jgdd}t|||gV  |j|jrRt|j|jrdtd S )Nrk   r   T)r   r  )	rZ   r<   r   r   r#   r  r   rh   r\   )r_   rR   rS   rT   r`   rb   rc   rd   rd   re   test_waiting_data  s    r  c       	      c   s  t | j| jfddV }t | j| jfddV }t| j|j|jhksFt|td}|td}|td}|j	|j	ksztt
||gV  | j|j|j	|j	h|j|j	hdt ikst| j|j	|jh|j	|j|jhikst| V  t }x0|j| jkrtdV  t |d k stqW |j| jks.t|j| j|j	 ksFt|j	| jksXt| V  t }x4| jrtdV  t |d k sjt| jqjW d S )NT)r   rk   r   zfire-and-forgetg{Gz?rg   )r!   r   r   r  Zclient_commsr  r\   rZ   r<   rh   r#   
wants_whatr  r   r,   r   r	   rt   )	rR   rS   rT   r_   rn   r`   rb   y2r   rd   rd   re   test_multi_client  s2    
&


r  c          	   C   s>   t  . t| }|dd d}|  td W d Q R X d S )Nc             S   s   | d S )Nrk   rd   )r`   rd   rd   re   r     s    z0long_running_client_connection.<locals>.<lambda>rV   r   )rK   r!   rZ   ra   r	   )r   r_   r`   rd   rd   re   long_running_client_connection  s
    r  c             c   s   t jt| jfd}d|_|  t }x(| jsNt	dV  t |d k s(t
q(W |  t }x(| jrt	dV  t |d k s`t
q`W d S )N)targetr   Tg{Gz?rg   )r1   Processr  r   Zdaemonr   r,   rt   r   r	   r\   Z	terminate)rR   rS   rT   procr   rd   rd   re   +test_cleanup_after_broken_client_connection
  s    r  c       	      c   sH  t | j| jfddV }t | j| jfddV }|td}|td}|td}|j|jks`tt||gV  |  t	 }x:|j|j
ks|j|j
krtdV  t	 |d k s~tq~W | j|j|jh|j|jhdt ikst| j|j|j|jhikst|  t	 }x:|j| j|j krDtdV  t	 |d k stqW tdV  |j|j
ksr|j|j
ksrt| j|j|jh|jt dt ikst| j|j|jhikst|  t	 }xB|j|j
ks|j|j
krtdV  t	 |d k stqW td	d
 | j D r$t| jr0t| V  | V  d S )NT)r   rk   r   g{Gz?rg   zfire-and-forgetg?c             s   s   | ]
}|V  qd S )Nrd   )ri   vrd   rd   re   rj   J  s    z0test_multi_garbage_collection.<locals>.<genexpr>)r!   r   r   rZ   r<   rh   r\   r#   r   r,   r  r   r	   r  r  r  r  r[  r\  r   )	rR   rS   rT   r_   rn   r`   rb   r  r   rd   rd   re   test_multi_garbage_collection  sH    
 
r   c             c   sF   | j ddgddV \}}|j|j  kr<|jd|jdiksBn td S )Nrk   r   T)r  )r   r  rh   r\   )r_   rR   rS   rT   r`   rb   rd   rd   re   test__broadcastQ  s    r  c             g   sP   | j ddgddV \}}t|j|j jdks2tt|j|j jdksLtd S )Nrk   r   )r  )r   rr   rt   rh   r   r\   )r_   rR   r   r`   rb   rd   rd   re   test__broadcast_integerW  s    r  c             c   s:   | j ddiddV }|j|j  kr0ddiks6n td S )Nr`   rk   T)r  )r   r  r\   )r_   rR   rS   rT   r6  rd   rd   re   test__broadcast_dict^  s    r  c             C   s   | j ddgdd\}}t| j| jj}dd | D |d |j|jh|d |j|jhiks`t| j dgd|d gd	\}t| j| jj}d
d | D |d |j|j|jh|d |j|jhikstd S )Nrk   r   T)r  c             S   s   i | ]\}}t ||qS rd   )r  )ri   r   r  rd   rd   re   
<dictcomp>i  s    z"test_broadcast.<locals>.<dictcomp>r   rl   )r  r   c             S   s   i | ]\}}t ||qS rd   )r  )ri   r   r  rd   rd   re   r  p  s    )r   r2   rO   ri  rj  itemsrh   r\   )r_   rR   rS   rT   r`   rb   rj  rc   rd   rd   re   test_broadcastd  s    r  c             c   s4   | j jddi|jdV }|d | d ks0td S )Nr  r
   )msgr  r  )ri  proxyr   r
   r\   )r_   rR   rS   rT   r  rd   rd   re   
test_proxyu  s    r	  c             c   s   |  td}|  t|}x|j|jkr4tdV  qW | |gV  | sPtdt	|ks`t|
  t }x*| stdV  t |d k sptqpW |jrt|
  d S )Nrk   g{Gz?cancelrg   )rZ   r8   rh   rt   r   r	   r
  	cancelledr\   r   r^   r,   )r_   rR   rS   rT   r`   rb   r   rd   rd   re   test__cancel{  s    

r  c          	   c   sB   | j tddd}|V }| |V  tt |V  W d Q R X d S )Nrk   )r`   r   rk   )rh   )rZ   r<   r
  r   r   r   )r_   rR   rS   rT   r`   ra   rd   rd   re   test__cancel_tuple_key  s
    r  c       	   	   c   s   t | j| jfddV }t | j| jfddV }|td}|td}|j|jksTt||gV  | snt| rztt	 }x.|j| j
krtdV  t	 |d k stqW |V }|dksttt |V  W d Q R X | V  | V  d S )NT)r   rk   g{Gz?rg   r   )r!   r   r   rZ   r8   rh   r\   r
  r  r,   rt   r   r	   r   r   r   r   )	rR   rS   rT   r_   rn   r`   rb   r   rv  rd   rd   re   test__cancel_multi_client  s$    
r  c             c   sr   |  tdgdgdgg}tdd t|D dd}| |V  | |gV  tdd |D sdt|jrntd S )	Nrk   r   rl   c             S   s   i | ]\}}|d |fqS )rT   rd   )ri   rl  rn   rd   rd   re   r    s    z+test__cancel_collection.<locals>.<dictcomp>rT   c             s   s   | ]}|  V  qd S )N)r  )ri   rn   rd   rd   re   rj     s    z*test__cancel_collection.<locals>.<genexpr>)	rp   rD   r   Bag	enumerater
  rs   r\   rt   )r_   rR   rS   rT   r   r`   rd   rd   re   test__cancel_collection  s    r  c             C   s   | j tddd}| j t|dd}| j t|dd}| |g t }x&| shtd t |d k sDtqDW | dkszt|  | std S )	Nrk   r`   )rh   rb   rc   g{Gz?rg   r   )rZ   r8   r
  r,   r  r	   r\   ra   )r_   r`   rb   rc   r   rd   rd   re   test_cancel  s    
r  c             c   s:   |  td}t|gV  |jtks&tdt|ks6td S )Nrk   rO  )rZ   r<   r#   r:  rO  r\   r   )r_   rR   rS   rT   r`   rd   rd   re   test_future_type  s    r  c       
   
   c   s   |  tdd}y
|V  W nh tk
r } zJ|}t \}}}	x2|	rnd|	jjjksTtd|	jjjksft|	j	}	q>W W d d }~X Y nX d S )Nrk   r   ri  r  )
rZ   r>   r   rw   r   tb_framef_codeco_filenamer\   r   )
r_   rR   rS   rT   r`   r  rn   r  r  r   rd   rd   re   test_traceback_clean  s    
r  c             c   s   ddl m}m} |dd}| t|}||s2t|jr<t| jt|dd}||sXt|jdksft| |V }	||	s~t|	d |	
 }
t|
tst|
V }|dkstd S )Nr   )rp  isqueuer   )ru  rl   rk   rm   )rq  rp  r  rp   r<   r\   ru  rD   Z_gatherrr  r   r]   r"   )r_   rR   rS   rT   rp  r  Zq_1Zq_2Zq_3Zq_4rn   ra   rd   rd   re   test_map_queue  s    


r  )rl   r   z!replace StopIteration with returnc             c   s   dd }|  dd | }t|ts(tt }x(|jsVtdV  t |d k s0tq0W | }y(x"t|}|V }	|	t|ksbtqbW W nR t	k
r }
 z4t
t	}t| W d Q R X |
j|jjkstW d d }
~
X Y nX d S )Nc               s   s   dV  dV  t dd S )Nrk   r   rl   )r{  rd   rd   rd   re   r   	  s    z(test_map_iterator_with_return.<locals>.gc             S   s   | S )Nrd   )r`   rd   rd   re   r   
	  s    z/test_map_iterator_with_return.<locals>.<lambda>g{Gz?rg   )rp   r]   r+   r\   r,   rt   r   r	   rm  r{  r   r   r   r  )r_   rR   rS   rT   r   f1r   Zg1rn   r   r  r   rd   rd   re   test_map_iterator_with_return	  s"    r  c             c   sh  t dddg}t dddg}| t||}t|ts8tt }x(|jsft	dV  t |d k s@tq@W | t
|}t|tstt|}	|	V }
|
d	kstt|}g }x|D ]}|V }|| qW |d
dgkstttd}| dd |}t|tstt|V }
|
dkstt|}g }x|D ]}|V }|| q,W |dd tddD ksdtd S )Nrk   r   rl   rV   rX   r   g{Gz?rg      ,   B   c             S   s   | S )Nrd   )r`   rd   rd   re   r   8	  s    z#test_map_iterator.<locals>.<lambda>)r   r   c             S   s   g | ]}||fqS rd   rd   )ri   rl  rd   rd   re   r   B	  s    z%test_map_iterator.<locals>.<listcomp>)iterrp   r   r]   r+   r\   r,   rt   r   r	   rD   rm  ry   r   r  rq   )r_   rR   rS   rT   r`   rb   r  r   f2r   ra   r   r   rn   r`  r  Z	futures_lrd   rd   re   test_map_iterator	  s<    


r!  c             C   s,   |  tddgtd}t|dks(td S )Nrk   r   rV   )rp   r   	itertoolsrepeatrr   r\   )r_   rR   rS   rT   r   rd   rd   re   test_map_infinite_iteratorsE	  s    r$  c             C   s0   t td}| dd |}t| dk d S )NrV   c             S   s   | S )Nrd   )r`   rd   rd   re   r   M	  s    z(test_map_iterator_sync.<locals>.<lambda>)r   r   )r  rq   rp   rm  ra   )r_   r  r   rd   rd   re   test_map_iterator_syncK	  s    r%  c             C   s(   t | tddgdddgdks$td S )Nrk   r   rl   )rr   rp   r   r\   )r_   rR   rS   rT   rd   rd   re   test_map_differnet_lengthsQ	  s    r&  c          
   C   s   t  >\}\}}t|d | d}tj |jks6tW d Q R X W d Q R X | \}}t|	 
ddkspttj |jkstd S )Nr   )rO   
rk   )r6   r!   r   baseget_schedulerr   r\   r  rr   stripsplit)rO   r  rR   rS   rT   r_   rv  r  rd   rd   re   test_Future_exception_sync_2V	  s    (r,  <   c             c   s  ddl m}m} |d}|t|}|t|}|t||}	| ||	g\}
}t|
t|ksdtt|t|	ksxtt	|
j
dkstt	|j
dkstt	|	j
dkst| |
 kst|	 | kstx(|j|jkr|	j|jkrtdV  qW |j|j | jhkst|j|	j | jhks4t| |
|g\}}| ||gV \}}|tdkslt|ttdtdkstt| ||stt| |gttfstd S )Nr   )r   Delayedrk   g{Gz?)r  r   r.  r<   r=   r   r   r:  r\   rr   r   __dask_keys__rh   rt   r   r	   r  r  r   rv   r]   ry   rX  )r_   rR   rS   rT   r   r.  r`   rb   rc   r  r  ZwwZyyfZwwfZyyyZwwwrd   rd   re   test_async_persista	  s.    r0  c             c   s   t d dd lm} |jddd}d|d  }t|jdksBt| |}t|jdks^tt|jdksptt	d	d
 |j
 D st| | kst| ||g\}}	| ||	gV \}
}|
|k	 std S )Nz
dask.arrayr   )rV   rV   )rg   rV   )r   r   rk   r   c             s   s   | ]}t |tV  qd S )N)r]   r"   )ri   r  rd   rd   re   rj   	  s    z test__persist.<locals>.<genexpr>)r   r   
dask.arrayarrayr   rr   r   r\   r   rs   r\  r/  r   rv   )r_   rR   rS   rT   r   r`   rb   r  r   hZggZhhrd   rd   re   test__persist	  s    

r4  c             C   s   t d dd lm} |jddd}d|d  }t|jdksBt| |}t|jdks^tt|jdksptt	d	d
 |j
 D st| | kst| }| }||k	 std S )Nz
dask.arrayr   )rV   rV   )rg   rV   )r   r   rk   r   c             s   s   | ]}t |tV  qd S )N)r]   r"   )ri   r  rd   rd   re   rj   	  s    ztest_persist.<locals>.<genexpr>)r   r   r1  r2  r   rr   r   r\   r   rs   r\  r/  r   )r_   r   r`   rb   r  r  rc   rd   rd   re   test_persist	  s    

r5  c             #   sj   ddl m}  fdd |  d}t|gV  t|| j|j jdk sNtt	| j|j j
tsftd S )Nr   )dumpsc                s"   | dkrdd  n | d S d S )Nr   rk   rd   )r   )deeprd   re   r7  	  s    
z!test_long_traceback.<locals>.deepr  i'  )Zdistributed.protocol.pickler6  rZ   r#   rr   r   rh   r  r\   r]   r   r   )r_   rR   rS   rT   r6  r`   rd   )r7  re   test_long_traceback	  s    r8  c             #   s^   |  tdgdgdgg}tdd t|D dd}t|V  t fdd|D sZtd S )	Nrk   r   rl   c             S   s   i | ]\}}|d |fqS )rT   rd   )ri   rl  rn   rd   rd   re   r  	  s    z,test_wait_on_collections.<locals>.<dictcomp>rT   c             3   s&   | ]}|j  jkp|j jkV  qd S )N)rh   r  )ri   rn   )rS   rT   rd   re   rj   	  s    z+test_wait_on_collections.<locals>.<genexpr>)rp   rD   r   r  r  r#   rs   r\   )r_   rR   rS   rT   r   r`   rd   )rS   rT   re   test_wait_on_collections	  s    
r9  c          	   C   s.  |  tdddg\}}}ttdt ks.ttt||hksDttt|||g|||hksdttt||g|ggg|||hksttt||gd||hksttdd t|||gD dd}tt||||hkstt|||t	t	t	||f|fd	fd
dd}tt||||hks*td S )Nrk   r   rl   r   )r`   rb   c             S   s   i | ]\}}|d |fqS )rT   rd   )ri   rl  rn   rd   rd   re   r  	  s    z'test_futures_of_get.<locals>.<dictcomp>rT   in)r`   rb   rc   rv  rv  )r:  )
rp   r<   r  r(   r\   r   r  r  r   r   )r_   rR   rS   rT   r`   rb   rc   Zsgrd   rd   re   test_futures_of_get	  s     & "r;  c              C   s"   t d} t| jgg kstd S )Nz
dask.array)r   r   r(   Arrayr\   )r   rd   rd   re   test_futures_of_class	  s    
r=  c          	   c   s   |  td}| |gV  tt |V  W d Q R X tt* | jt|ftdfdddgddV  W d Q R X tt |  t| W d Q R X tt | j td|d W d Q R X tt | jtdg|d W d Q R X d|j	kst
d S )	Nrk   r   )r`   rb   r`   rb   F)r2   )rb   )rZ   r<   r
  r   r   r   r   r   rp   rt   r\   )r_   rR   rS   rT   r`   rd   rd   re    test_futures_of_cancelled_raises	  s    .r>  c             c   s   |  td}t|gV  |  tjV  |  td}t }x.|j|jkrht	dV  t |d k s<t
q<W x8t ||jd d  k r|j|jkst
t	dV  qlW d S )Nrk   g{Gz?r   i  )rZ   r<   r#   r   r   r   r,   rh   r  r	   r\   Zdelete_interval)r_   rR   r  r`   r   r   rd   rd   re   #test_dont_delete_recomputed_results	  s    r?  c             c   s.   t  }| t|}x|js(tdV  qW d S )Ng{Gz?)r  rZ   r<   rt   r   r	   )r_   rR   r  r   rd   rd   re   test_fatally_serialized_input
  s    r@  zUse fast random selection nowc             c   sV   |  td}t|V  |  td}t|V  t|jt|j  krLdksRn td S )Nrk   r   )rZ   r<   r#   rr   r  r\   )r_   rR   rS   rT   r`   rb   rd   rd   re   test_balance_tasks_by_stacks
  s
    

rA  c             c   sp   |  tdV }||jd|jdiks&t| j td|jgdV }||jdiksNt| j tdg dV }|i ksltd S )Nrk   r   )r   )runr<   r   r\   )r_   rR   rS   rT   r   rd   rd   re   test_run
  s    rC  c             #   sj   |  ttd}t|V  dd  |  V }|  ks<t|  V }| fdd||gD ksftd S )NrV   c               S   s   i t  g ddddfS )Nrd   rk   hellos   100)r  rd   rd   rd   re   r   )
  s    z-test_run_handles_picklable_data.<locals>.funcc                s   i | ]}  |j qS rd   )r   )ri   r  )r   rd   re   r  0
  s    z3test_run_handles_picklable_data.<locals>.<dictcomp>)rp   r<   rq   r#   run_on_schedulerr\   rB  )r_   rR   rS   rT   r   r   rd   )r   re   test_run_handles_picklable_data$
  s    
rF  c             C   sf   d
dd}| j |ddd}||d d|d diks6t| j |dd|d gd	}||d diksbtd S )NrV   c             S   s   | | S )Nrd   )r`   rb   rd   rd   re   r   4
  s    ztest_run_sync.<locals>.funcrk   r   )rb   r   rl   )rb   r   )rV   )rB  r\   )r_   rR   rS   rT   r   ra   rd   rd   re   test_run_sync3
  s    

rG  c          	   c   s   | j tdddV }||jd|jdiks*t| j tdd|jgdV }||jdiksTt| j tdg dV }|i ksrttt}|  tdV  W d Q R X dt|kstt	j
dkr| j tdd	dV }||jd
|jd
ikstd S )Nrk   g?)r   r   )r   r   )r   rD  )rl   rg   g{Gz?rl   )rB  r@   r   r\   r   r   RuntimeErrorr?   r   rw   rx   rA   )r_   rR   rS   rT   r   r   rd   rd   re   test_run_coroutine?
  s    
rI  c             C   s   | j tddd}||d d|d diks,t| j td|d gd}||d diksVtt }| j tdddd	}t }|d kst|| d
kstd S )Nr   g{Gz?)r   r   rl   )r   rV   F)r   r#   g      ?)rB  r@   r\   r,   )r_   rR   rS   rT   ra   Zt1Zt2rd   rd   re   test_run_coroutine_syncS
  s    
rJ  c          
   C   sT   dd }xFt tgD ]:}t|}| ||d W d Q R X dt|jkstqW d S )Nc             S   s   | |d S )Nrd   )r  Zexc_msgrd   rd   re   raise_exceptionc
  s    z+test_run_exception.<locals>.raise_exceptionzinformative message)r$  rH  r   r   rB  r   r  r\   )r_   rK  r  excinford   rd   re   test_run_exceptionb
  s
    rM  c          
      s  t  \}\}}|d  |d t|d | d}| }| ddiksRt| g}| diksnt| }| dikst||d }| dikst|td|td|tdtg | }t|j	j	j	hkstt
 fdd| D stt
| s0t|g}t|j	j	hksXt|}t|j	hksxt| }t| hkstt
fdd| D st| g}t| hkst| }t| hkstW d Q R X W d Q R X d S )	Nr   )rO   rk   r   rl   c             3   s$   | ]}|D ]}| gkV  q
qd S )Nrd   )ri   r  r  )a_addrb_addrrd   re   rj   
  s    z%test_diagnostic_ui.<locals>.<genexpr>c             3   s,   | ]$}|D ]}| j j j gkV  q
qd S )N)rh   )ri   r  r   )r`   rb   rc   rd   re   rj   
  s    )r6   r!   r   r\   rZ   r<   r#   r   r  rh   rs   r\  rj  )rO   rR   rS   rT   r_   r6  rd   )rN  rO  r`   rb   rc   re   test_diagnostic_uil
  s>    
"
$
rP  c             C   s   |  tdddg}|  tdddg}t||  | jdddd || D ksRt| jddtdd tdd d	ks|td S )
Nrk   r   rl   F)r4  c             S   s   i | ]}t d |jqS )rk   )r/   rh   )ri   r   rd   rd   re   r  
  s   z/test_diagnostic_nbytes_sync.<locals>.<dictcomp>T)r<   rD   )rp   r<   rD   r#   nbytesr\   r/   )r_   incsdoublesrd   rd   re   test_diagnostic_nbytes_sync
  s    rT  c             c   s   |  tdddg}|  tdddg}t|| V  |jdddd || D ksTt|jddtdd tdd d	ks~td S )
Nrk   r   rl   F)r4  c             S   s   i | ]}t d |jqS )rk   )r/   rh   )ri   r   rd   rd   re   r  
  s   z*test_diagnostic_nbytes.<locals>.<dictcomp>T)r<   rD   )rp   r<   rD   r#   r5  r\   r/   )r_   rR   rS   rT   rR  rS  rd   rd   re   test_diagnostic_nbytes
  s    rU  c        
      c   sV  t dd} | d t| j| jdd}t| j| jdd}t| j| jdd}| | | gV  t| j| jfddV }|jtt	d	dd
}|j
ddd
V }t|V  t|jd	kstt|jdkstt|j|jdikstxBtddgdgD ].\}}|jdd ||d
V }	|	|d kstqW | V  | | | gV  |  V  d S )NT)r  r   Zalice)rg  Zbobrl   )r   rV   )r   rN  c             S   s   | d S )Nrk   rd   )r`   rd   rd   re   r   
  s    z%test_worker_aliases.<locals>.<lambda>rk   )r-   r   r   r   r   _startr!   rp   r<   rq   r   r#   rr   r  r\   r8  rh   r  rZ   r   r   )
rR   rS   rT   r  r_   r   r   rl  aliasra   rd   rd   re   test_worker_aliases
  s&    



rX  c       	      C   sp   t t}t dt d }}t t||}t t||}t t||}| |}t t|d}| dksltd S )Nrk   r   rV      )r   r   r   r   r\   )	r_   daddr`   rb   r   r  xxyyxxyy2xxyy3rd   rd   re   test_persist_get_sync
  s    
r^  c             c   s   t t}t dt d }}t t||}t t||}t t||}	| |	}
t t|
d}tdV  | j|j| ddV }|d dkst| 	|V }|dkst| 	|V }|dkst| 	|V }|dkstd S )	Nrk   r   rV   g      ?F)r2   r   rY  )
r   r   r   r   r	   r   r   r/  r\   r   )r_   rR   rS   rT   rZ  r`   rb   r   r  r[  r\  r]  ra   rd   rd   re   test_persist_get
  s     
r_  winz num_fds not supported on windowsc       
      C   s   t d}t \}\}}| }t|d | dn}| }x,tdD ] }t|d | d W d Q R X qFW t }	x*| |krtd t |	d k srt	qrW W d Q R X W d Q R X d S )Npsutilr   )rO   rm   g{Gz?)
r   r   r6   r  r!   num_fdsrq   r,   r	   r\   )
rO   ra  rR   rS   rT   r  r_   beforerl  r   rd   rd   re   test_client_num_fds
  s    
rd  c             c   sD   t | j| jfddV }| V  t | j| jfddV }| V  d S )NT)r   )r!   r   r   r   )rR   rS   rT   r_   rd   rd   re   test_startup_close_startup
  s    
re  c          
   C   s   t  \}\}}t|d | d}td W d Q R X t|d }W d Q R X t|d }W d Q R X td t|d }W d Q R X W d Q R X d S )Nr   )rO   g?)r6   r!   r	   )rO   rR   rS   rT   r_   rd   rd   re   test_startup_close_startup_sync  s    

rf  c          
   c   s\   dd }|  |}y
|V }W n2 tk
rN } zdt|ks>tW d d }~X Y n
X dsXtd S )Nc              S   s   G dd dt } | dd S )Nc               @   s   e Zd Zdd ZdS )zMtest_badly_serialized_exceptions.<locals>.f.<locals>.BadlySerializedExceptionc             S   s
   t  d S )N)r   )r  rd   rd   re   
__reduce__  s    zXtest_badly_serialized_exceptions.<locals>.f.<locals>.BadlySerializedException.__reduce__N)r@  rA  rB  rg  rd   rd   rd   re   BadlySerializedException  s   rh  zhello world)r   )rh  rd   rd   re   rn     s    z+test_badly_serialized_exceptions.<locals>.fzhello worldF)rZ   r   r   r\   )r_   rR   rS   rT   rn   r`   ra   r  rd   rd   re    test_badly_serialized_exceptions  s    

"ri  c             c   s*  |j |j }|j |j }| jddg|jgdV \}}t|jdksFtt|jdksXt|  |  V  |  t|jdkstdd |jD t	|jkst||j
|j jks||j
|j jkstt|jdkstdd |jD t	|jkst||j
|j jks&||j
|j jks&td S )Nrk   r   )r   r   c             S   s   h | ]
}|j qS rd   )rh   )ri   rZ  rd   rd   re   r   4  s    z!test_rebalance.<locals>.<setcomp>c             S   s   h | ]
}|j qS rd   )rh   )ri   rZ  rd   rd   re   r   8  s    )r   r   r   rr   r  r\   r^   	rebalancerj  r  rt   rh   r   )r_   rR   rS   rT   r  r  r`   rb   rd   rd   re   test_rebalance&  s    
(rk  c       
      c   sZ  | j ddddg|jgdV \}}}}	t|jdks6tt|jdksHtt|jdksZtt|jdkslt| j||g|j|jgdV  t|jdkstt|jdkstt|jdkstt|jdkst|j|jdiks|j|jdikst|  V  t|jdkstt|jdks&tt|jdks:tt|jdksNt|  d S )Nrk   r   rl   rm   )r   r   )r   r   rr   r  r\   rj  rh   r^   )
r  rR   rS   rT   r_   r6  r  r`   rb   rc   rd   rd   re   test_rebalance_workers=  s"    $$
rl  c             c   sT   | j ttd|jd}| |V  t|jt|j  krBdksHn t|  d S )NrV   )r   rg   )	rp   r<   rq   r   rj  rr   r  r\   r^   )r_   rR   rS   rT   r   rd   rd   re   test_rebalance_executionT  s    &rm  c             C   s^   | j ttd|d gd}| | |  }t|dks<tttt|	 ddgksZtd S )NrV   r   )r   r   rg   )
rp   r<   rq   rj  rj  rr   r\   ry   r   r\  )r_   rR   rS   rT   r   rj  rd   rd   re   test_rebalance_sync\  s
    
rn  c             c   s<   | j ttdd|jd}tdV  | |V  |  d S )NrV   g?)r   r   g?)rp   r8   rq   r   r   r	   rj  r^   )r_   rR   rS   rT   r   rd   rd   re   test_rebalance_unpreparede  s    ro  c             c   s\   | j td|jgd}|V }| V  t }x,|jdkrVt |d k sHttdV  q,W d S )Nrk   )r   r  rg   g{Gz?)	rZ   r<   r   r   r,   r   r\   r   r	   )r_   rR   rS   rT   r`   ra   r   rd   rd   re   test_receive_lost_keym  s    
rp  c             c   s  | j td|jgd}|V }| V  t }x,|jdkrVt |d k sHttdV  q,W |j	|j
 |jksnt|j|j
gd|j
dikstt|j|j|jd}| V  t }x,|jdkrt |d	 k sttdV  qW |j	|j
 |jkst|V }|d	kst| V  d S )
Nrk   )r   r  rg   g{Gz?)r   z	no-worker)rO   r   )rZ   r<   r   r   r,   r   r\   r   r	   rt   rh   Z
unrunnableget_task_statusr   r   rO   rV  )r_   rR   rS   rT   r`   ra   r   r  rd   rd   re   test_unrunnable_task_runsy  s&    

rr  c             c   sJ   |  ttd}t|j|jd|jd}|d | |V }|	 V  d S )NrV   r   )r   rO   r   )
rp   r<   rq   r   r   r   rO   r   rv   r   )r_   rR   r   r   ra   rd   rd   re   test_add_worker_after_tasks  s
    
rs  c             c   s   | j dg|jdV \}| jt||jd}|V  |j|j dksBt|j|j j	|j
|j |j
|j hkslt|j
|j j|j|j |j|j hkst|  d S )Nrk   )r   )r   r   rZ   r<   r   r  rh   r\   rt   r   r   rj  r^   )r_   rR   rS   rT   r`   rb   rd   rd   re   #test_workers_register_indirect_data  s    *rt  c          	   c   sD   |  td}|V  | |V  tt |  t|}W d Q R X d S )Nrk   )rZ   r<   r
  r   r   r   )r_   rR   rS   rT   r`   rb   rd   rd   re   test_submit_on_cancelled_future  s
    ru  rV   c             '   s   |  ddgV \ |j jjgddV  |  t|j j jdksNtt|jj jdkshtt fdd|D dksttfdd|D dkstd S )Nrk   r   rg   )r   r   c             3   s   | ]} j |jkV  qd S )N)rh   r  )ri   r  )rS   rd   re   rj     s    z!test_replicate.<locals>.<genexpr>c             3   s   | ]} j |jkV  qd S )N)rh   r  )ri   r  )rT   rd   re   rj     s    )	r   	replicaterh   r^   rr   rt   r   r\   ru   )r_   rR   r   rd   )rS   rT   re   test_replicate  s    rw  c             c   sZ   t tddd}| |}| j|ddV  |  |jr>|jsBt| |V  |  d S )Nrk   )r`   rk   )r  rg   )r   )r   r<   r   rv  r^   r  r\   rj  )r_   rR   rS   rT   r`   rn   rd   rd   re   test_replicate_tuple_keys  s    
rx  c             '   s  | j ddg|d jgdV \ |j jjgddd |d d D dV  t|j j jdksftt|jj jdkstt fd	d
|d d D dksttfdd
|d d D dkstt fdd
|dd  D dksttfdd
|dd  D dkst|j jjgddV  t|j j jdksNtt|jj jdksjtt fdd
|D dksttfdd
|D dkst|	  |j jjgd dV  t|j j jdkstt|jj jdkst|	  |j jjgddd |d d D dV  t fdd
|d d D dks^ttfdd
|d d D dkstt fdd
|dd  D dksttfdd
|dd  D dkst|	  d S )Nrk   r   r   )r   rg   c             S   s   g | ]
}|j qS rd   )r   )ri   r  rd   rd   re   r     s    z*test_replicate_workers.<locals>.<listcomp>)r   r   r   c             3   s   | ]} j |jkV  qd S )N)rh   r  )ri   r  )rS   rd   re   rj     s    z)test_replicate_workers.<locals>.<genexpr>c             3   s   | ]} j |jkV  qd S )N)rh   r  )ri   r  )rT   rd   re   rj     s    c             3   s   | ]} j |jkV  qd S )N)rh   r  )ri   r  )rS   rd   re   rj     s    c             3   s   | ]} j |jkV  qd S )N)rh   r  )ri   r  )rT   rd   re   rj     s    )r   r   c             3   s   | ]} j |jkV  qd S )N)rh   r  )ri   r  )rS   rd   re   rj     s    c             3   s   | ]} j |jkV  qd S )N)rh   r  )ri   r  )rT   rd   re   rj     s    rV   c             S   s   g | ]
}|j qS rd   )r   )ri   r  rd   rd   re   r     s    c             3   s   | ]} j |jkV  qd S )N)rh   r  )ri   r  )rS   rd   re   rj     s    c             3   s   | ]} j |jkV  qd S )N)rh   r  )ri   r  )rT   rd   re   rj     s    c             3   s   | ]} j |jkV  qd S )N)rh   r  )ri   r  )rS   rd   re   rj     s    c             3   s   | ]} j |jkV  qd S )N)rh   r  )ri   r  )rT   rd   re   rj     s    )
r   r   rv  rh   rr   rt   r   r\   ru   r^   )r_   rR   r   rd   )rS   rT   re   test_replicate_workers  s4     &&&(  ((((ry  c               @   s$   e Zd Zdd Zdd Zdd ZdS )CountSerializationc             C   s
   d| _ d S )Nr   )r   )r  rd   rd   re   __init__  s    zCountSerialization.__init__c             C   s   |d | _ d S )Nrk   )r   )r  r   rd   rd   re   r    s    zCountSerialization.__setstate__c             C   s   | j S )N)r   )r  rd   rd   re   r    s    zCountSerialization.__getstate__N)r@  rA  rB  r{  r  r  rd   rd   rd   re   rz    s   rz  c             '   sP   t  }| |gV \ |j jgddV  t fdd|D }|dksLtd S )NrV   )r   r   c             3   s   | ]}|j  j jV  qd S )N)r  rh   r   )ri   r  )r   rd   re   rj     s    z0test_replicate_tree_branching.<locals>.<genexpr>rk   )rz  r   rv  rh   maxr\   )r_   rR   r   rF  Z	max_countrd   )r   re   test_replicate_tree_branching  s
    r}  c             g   s   |  td}|  td}| j||gddV  t|j|j jdksFtt|j|j jdks`t| j||gddV  t|j|j jdkstt|j|j jdkst| ||gV  |  t|j|j jdkstt|j|j jdkstd S )Nrk   r   rg   )r   rl   rV   )	rZ   r<   rv  rr   rt   rh   r   r\   r^   )r_   rR   r   r`   rb   rd   rd   re   test_client_replicate  s    r~  )z	127.0.0.2rk   )rL   r   rU   c       	      c   s   |j |j }|j |j }|j |j }| jtddd}t|gV  |j|j j|hksl|j|j j|hkslt| j	|gdgdV  |j|j j||hkst| j	|gdgdV  |j|j j|||hkstd S )Nrk   z	127.0.0.2)r   z	127.0.0.1)
r   r   rZ   r<   r#   rt   rh   r   r\   rv  )	rL   rR   rS   rT   r_   r  r  Zcwsr`   rd   rd   re   test_client_replicate_host%  s    r  c          	   C   s   |  td}|  td}| j||gdd |  }t||j t||j   krZdks`n ttt	 | j|gdd W d Q R X |
 dkstd S )Nrk   r   )r   r   rl   )rZ   r<   rv  r   rr   rh   r\   r   r   r$  ra   )r_   r`   rb   r   rd   rd   re   test_client_replicate_sync;  s    .r  z Windows timer too coarse-grained)z	127.0.0.1rm   rk   c             c   s|   | j tddd}t|V  d|jd   k r4dk s:n t| jttddd}t|V  d|jd   k rrd	k sxn td S )
Nrk   g?)r   g333333?r8   g?rV   r   g?)rZ   r8   r#   ra  r\   rp   rq   )r_   rR   rS   r   r   rd   rd   re   test_task_load_adapts_quicklyI  s    
 
r  c             #   s   | j td jd}| j tdjd}t||gV  | ttdd}t|V  t fdd|D shttfdd|D std S )Nrk   )r   r   rW   c             3   s   | ]}|j  jkV  qd S )N)rh   r  )ri   rn   )rS   rd   re   rj   _  s    z6test_even_load_after_fast_functions.<locals>.<genexpr>c             3   s   | ]}|j  jkV  qd S )N)rh   r  )ri   rn   )rT   rd   re   rj   `  s    )rZ   r<   r   r#   rp   rq   r[  r\   )r_   rR   rS   rT   r`   rb   r   rd   )rS   rT   re   #test_even_load_after_fast_functionsW  s    
r  c             c   sL   |  tddg\}}t||gV  t|jt|j  krBdksHn td S )Nrk   r   )rp   r<   r#   rr   r  r\   )r_   rR   rS   rT   r`   rb   rd   rd   re   test_even_load_on_startupe  s    r  )z	127.0.0.1r   c       	      c   sn   |  tddddg\}}}}t||||gV  t|jt|jg}|j|jh|ksVt|j|jh|ksjtd S )Nrk   r   rl   rm   )rp   r<   r#   r  r  rh   r\   )	r_   rR   rS   rT   r  r`   rb   rc   groupsrd   rd   re   test_contiguous_loadl  s
    r  c             '   sD    fddt dD }t|V  x|D ]}t|jdks&tq&W d S )Nc                s   g | ]}  t|qS rd   )rZ   r8   )ri   rl  )r_   rd   re   r   y  s    z-test_balanced_with_submit.<locals>.<listcomp>rm   rk   )rq   r#   rr   r  r\   )r_   rR   r   r   r  rd   )r_   re   test_balanced_with_submitw  s    

r  c             '   sZ    j dgddV \ fddtdD }t|V  x|D ]}t|jdks<tq<W d S )NrV   T)r  c                s   g | ]} j td dqS )F)r   )rZ   r8   )ri   rl  )r_   r`   rd   re   r     s    z?test_balanced_with_submit_and_resident_data.<locals>.<listcomp>rm   r   )r   rq   r#   rr   r  r\   )r_   rR   r   r   r  rd   )r_   r`   re   +test_balanced_with_submit_and_resident_data  s
    

r  )z	127.0.0.1rX   c             c   st   xndD ]f}| j ttd|d}| j t||d d}x8|jsj|jr\tdd |j D s\tt	dV  q4W qW d S )N)r   g{Gz?g?r   )r   rV   c             s   s*   | ]"}|j  D ]}t|d kV  qqdS )rX   N)
processingr\  rr   )ri   r  prd   rd   re   rj     s   z1test_scheduler_saturates_cores.<locals>.<genexpr>g{Gz?)
rp   r8   rq   rt   rs   r   r\  r\   r   r	   )r_   rR   rS   rT   r   r   rd   rd   re   test_scheduler_saturates_cores  s    
r  c             c   s`   xZdD ]R}| j ttddd}x8|jsV|jrHtdd |j D sHtt	dV  q W qW d S )N)r   g{Gz?g?r   g?)Zscalec             s   s*   | ]"}|j  D ]}t|d kV  qqdS )rX   N)r  r\  rr   )ri   r  r  rd   rd   re   rj     s   z8test_scheduler_saturates_cores_random.<locals>.<genexpr>g{Gz?)
rp   r;   rq   rt   rs   r   r\  r\   r   r	   )r_   rR   rS   rT   r   r   rd   rd   re   %test_scheduler_saturates_cores_random  s    
r  c             g   s   t d}| jtddd}x|js0tdV  qW | |V  t }x:t	dd |j
 D r~t |d k spttdV  qFW |  d S )Nz
dask.arrayrk   g?)r   g{Gz?c             s   s   | ]}|j D ]
}|V  qqd S )N)r  )ri   r  r  rd   rd   re   rj     s    z0test_cancel_clears_processing.<locals>.<genexpr>)r   r   rZ   r8   rt   r   r	   r
  r,   r[  r   r\  r\   r^   )r_   rR   r   r   r`   r   rd   rd   re   test_cancel_clears_processing  s    
r  c              C   sL  t  :\} \}}tj }tttjjd t	| d dd.}tj |jksTt
tjddksht
W d Q R X tj |kst
tttjjd t	| d dd}tj |kst
tttjjd |  t	| d dd}tjddkst
tj |jkst
|  tj |ks(t
tttjjd t	| d }tj |jks^t
W d Q R X t	| d dd}tj |jkst
W d Q R X tj |jkst
t	| d ddd}tj |jkst
t	| d dd}tj |jks t
W d Q R X tj |jks t
W d Q R X tj |ks>t
W d Q R X d S )NZshuffler   T)Zset_as_defaultrt   F)r6   r   r(  r)  r   r   KeyErrorconfigr   r!   r\   r   )rR   rS   rT   Zpre_getr_   c1r|  rd   rd   re   test_default_get  s:    
    r  c             c   s   |   V }|tt|j kst| jttdd|jgdd}t	dV  |   V }t
||j|jhksht| j |jgdV }t||j ttfstd S )NrV   g?T)r   r   r  g?)r   )r  r   rX  r\   rp   r8   rq   r   r   r	   r  r]   ry   )r_   rR   rS   rT   r  r   r`   rd   rd   re   test_get_processing  s    

r  c             c   s  |  ttd}t|V  | j V }||jks4t| jj|jgdV }||j|j|j iksbt| j V }t	t
|t	t
|jkst| jj|jgdV }t	t
||jt
|j|j ikst| jjddV }||jddkst| jj|d jgddV }||d j|j|d j jiks$t| j V }t	t
|t	t
|jksLt| jj|d jgdV }t	t
||d jt
|j|d j ikstd S )NrV   )r   F)r4  r   )r   r4  )r   )rp   r<   rq   r#   ri  r   r\   r   rj  r   r  rQ  r5  rh   rt   r   )r_   rR   rS   rT   r   r`   rd   rd   re   test_get_foo  s$    
$&r  c             C   sV   t |  t | kstx4| D ](}| | }|| }t|t|ks&tq&W d S )N)r  r   r\   ry   )ZexpectedZactualr   Zevavrd   rd   re   assert_dict_key_equal  s
    r  rl   c             c   s  | j td|jgd}| jd|jgdV }t||gV  |j|j|j  }}}	| j V }
t|
||jg||jg|	g i | jj||gdV }
t|
||jg||jgi | j	 V }
t|
|j|g|j|gi | jj	|j|jgdV }
t|
|j|g|j|gi |
 V  |
 V  | j V }
t|
|	g i | jj||gdV }
t|
|g |g i | j	 V }
t|
|jg i | jj	|j|jgdV }
t|
|jg |jg i d S )Nrk   )r   rl   )r   )rZ   r<   r   r   r#   ri  rj  r  rh   r   r   )r_   rR   ur  r  r`   rb   ZuaZvaZwar6  rd   rd   re   test_get_foo_lost_keys  s,    

r  )rL   r   check_new_threadsc          	   c   s.   |  tjd}tt |V  W d Q R X d S )Nrk   )rZ   rw   r  r   r   r.   )r_   rR   rS   rT   rn   rd   rd   re   test_bad_tasks_fail.  s    r  c             C   s   |   }tdd | D r"t| jttdd|d gdd}td |d }|d }|   }t|  ||hksxtt|  |g|hkst| 	| d S )	Nc             s   s   | ]
}|V  qd S )Nrd   )ri   r  rd   rd   re   rj   8  s    z+test_get_processing_sync.<locals>.<genexpr>rV   g?r   F)r   r   r  g?)
r  r[  r\  r\   rp   r8   rq   r	   r  r
  )r_   rR   rS   rT   r  r   rn  Zbbrd   rd   re   test_get_processing_sync6  s    r  c             C   s   |    |    |    d S )N)r   )r_   rd   rd   re   test_close_idempotentJ  s    r  c          	      s   t  }tt$  tdftdfdddg}W d Q R X t  |d k sJtt fdddd	 t fd
ddd	  t	d}|
  tt*  dtdf|jt	dfid|jg}W d Q R X |j jkstd S )Nrk   )r`   rb   r`   rb   g      ?c                  s    j  S )N)r   rd   )r_   rd   re   r   W  s    z(test_get_returns_early.<locals>.<lambda>g?)rU   c                  s   t     S )N)r[  r  r\  rd   )r_   rd   re   r   Y  s    rl   )r,   r0   rH  r   r?   r	   r\   rI   rZ   r<   ra   rh   r   )r_   r   ra   r`   rd   )r_   re   test_get_returns_earlyP  s    
(
.r  )r   rL   c             c   sh   |  td}|j| jkst|  V  |j| jks6t|j}~dd l}|  tj	V  || jksdtd S )Nrk   r   )
rZ   r<   rh   r	  r\   Zrestartr   r   r   r   )r_   rR   rS   rT   r`   rh   r   rd   rd   re   +test_Client_clears_references_after_restartc  s    
r  c          	   C   sn   t t$ | tdftdfdddg W d Q R X t }x.t|  	 rhtd t |d k s<t
q<W d S )Nrk   g      ?)r`   rb   r`   rb   g{Gz?g      ?)r   r   rH  r   r?   r	   r,   r[  r  r\  r\   )r_   r   rd   rd   re   test_get_stops_work_after_erroru  s    (r  c             C   sD   |  tttd}tt|}t| |dddddhks@td S )Nrg   rk   r   rl   rm   )	rp   r<   r  rq   ry   r$   r  rv   r\   )r_   r;  seq2rd   rd   re   test_as_completed_list  s    r  c             C   s^   |  ttd}tt|dd}ttd|dddddhks@tttd|t|ksZtd S )	Nrg   T)with_resultsrk   r   rl   rm   r   )rp   r<   rq   ry   r$   r  r   r\   )r_   r;  r  rd   rd   re   test_as_completed_results  s     r  r  c             C   s   d}| j tt|dd}g }x<t||d D ](}t|ttfsDtt	d |
| q.W t||ksjt|rttd|ttd|d kstnt|t|kstd S )N2   g{Gz?)r   )r  g?rk   )rp   r8   rq   r$   Zbatchesr]   rX  ry   r\   r	   extendrr   r  r   )r_   r  r   r   rv  Zbatchrd   rd   re   test_as_completed_batches  s    &r  c             C   sN   | j ttddd}t|}|jddg ks0tt|jdd|sJtd S )Nr   g?)r   F)blockT)rp   r8   rq   r$   Z
next_batchr\   r  r^  )r_   r   r  rd   rd   re   test_as_completed_next_batch  s    r  c              c   sf   t  } | d t| j| jfddV }|jdks4t|td}|	 V  |jdksXt| 	 V  d S )Nr   T)r   runningrk   closed)
r-   r   r!   r   r   r   r\   rZ   r<   r   )rR   r_   r`   rd   rd   re   test_status  s    

r  c       	      #   s   d}x| j | jgD ]}tj|dd}|d7 }|t}|t}||dd}t|V  ttt|	 
 jstttj|dd}|d7 }|t}|t}||dd}t|V  t fdd	|	 D rtqW d S )
NrV   r   )r   rk   F)r   Tc             3   s   | ]}t | jkV  qd S )N)r4   rt   )ri   r   )rR   rd   re   rj     s    z.test_persist_optimize_graph.<locals>.<genexpr>)r   r   r   rq   rp   r<   r#   r  r4   r/  r^  rt   r\   r[  )	r_   rR   rS   rT   rl  methodZb2Zb3Zb4rd   )rR   re   test_persist_optimize_graph  s     





r  c          	   c   s,   t tj | jdddV  W d Q R X d S )Nrk   g      ?)rU   )r   r   r   r   r   )r_   rR   rd   rd   re   !test_scatter_raises_if_no_workers  s    r  c          	   C   s@  t dd| d}|  ddddddg}t|d}td	| d}t }x.t| d
krptd t |d k sDtqDW |	t
d
}| dkstW d Q R X t }x(|jdkrt |d k sttd qW tt |  W d Q R X |jdksttt |  W d Q R X t|}t }x.|jdkrPtd t |d k s$tq$W t }x4t| d
krtd t |d k sZtqZW |	t
d
}| dkstW d Q R X t }xjy|  dstW n0 tk
r   wY n tk
r   P Y nX t |d k sttd qW t| |j |  d S )Nz	127.0.0.1i$  )rO   zdask-schedulerz--hostz--portZ9393z
--no-bokehz127.0.0.1:9393rk   g?rl   r   Z
connectingrg   g{Gz?r  r  g?r  F)r   r   rE   r!   r,   rr   r   r	   r\   rZ   r<   ra   r   r   r   r   r   r    r2   r   r   )rO   r  Zscheduler_clirR   r_   r   r`   rd   rd   re   test_reconnect  sZ    


r  z0Semaphore.acquire doesn't support timeout optionzTODO: intermittent failureszworker,count,repeatr   rX   c          
      sj  t d}| }tddd\}t  | }td t	 t
jd fdd	}x2t|D ]&}	j|t d t d |d	 qjW td
 dj}
td x*t|D ]}	 jdd t  sP qW t }x(|
 rtd t |d k stqW W d Q R X W d Q R X t }x@| |krdtd||  td t |d k s&tq&W d S )Nra  r   rX   )r  Zactive_rpc_timeoutrk   c             3   sz   xlt |D ]`}t| V  d d}d |< | V  |j}||< t|V  | V  ~tjV  q
W    d S )Nr   )rO   )rq   r   r	   rV  Zworker_addressr   r   r   )r	   Zdurationr#  rl  r  Zaddr)r[   rO   r  rR   r  rd   re   start_worker   s    

z2test_open_close_many_workers.<locals>.start_workerrg   )r#  r   )rO   )rU   g?rV   zfds:g?)rk   )r   r   r  r6   r   r   rb  r   r   WeakKeyDictionaryr   	coroutinerq   Zadd_callbackr.  r!   r	   acquirer,   r   r\   r  )rO   r  countr#  ra  r  r   rc  r  rl  r_   r   rd   )r[   rO   r  rR   r  re   test_open_close_many_workers  s8    
*r  c             c   sl  t | j| jfddV }t | j| jfddV }|td}|V  t| j}t|}|td}|j|jkslt	|V  t
dV  t| j}	||	kst	|tdd}t|V  |jdkst	t| j}|tdd}|j|jkst	t|V  t
dV  t| j}	||	kst	| j  |td}
|td}t|
|gV  t| j|ksTt	| V  | V  d S )NT)r   rk   g?r   r  r   )r!   r   r   rZ   r<   ry   transition_logrr   rh   r\   r   r	   r>   r#   r   clearr   )rR   rS   rT   r_   rn   r`   logZlen_single_submitrb   Zlog2r6  r  rd   rd   re   test_idempotenceH  s:    







r  c             C   s.   |   }t|tstt|d dks*td S )Nr   r   )scheduler_infor]   r8  r\   rr   )r_   inford   rd   re   test_scheduler_infot  s    r  c          
   C   s|   |   }tdb}| | t|d"}|  }| jj|jjksDtW d Q R X tt	 | | W d Q R X W d Q R X d S )NZjson)scheduler_file)
r  r5   Zwrite_scheduler_filer!   ri  r   r\   r   r   r$  )r_   r  r  r|  Zinfo2rd   rd   re   test_write_scheduler_filez  s    

r  c             C   s   t d}|  }|d d k	s"t|d d k	s2tt|d dksFtx$|d  D ]\}}|d k	sTtqTW | jdd | jdgd}t|d d	 d
 d |jkstd S )Nrequestsri  rL   r   r   T)Zcheck)packagesr  Zoptional)r   r   Zget_versionsr\   rr   r  r8  __version__)r_   r  r  r   rd   rd   re   test_get_versions  s    
r  c                sR   dd l }xD|jj|jj|jjgD ]*  fdd}| |}| dks tq W d S )Nr   c                  s    ddd fidS )Nr`   c               S   s   dS )Nrk   rd   rd   rd   rd   re   r     s    zAtest_threaded_get_within_distributed.<locals>.f.<locals>.<lambda>rd   rd   )r   rd   re   rn     s    z/test_threaded_get_within_distributed.<locals>.frk   )	Zdask.multiprocessingrw  Zget_syncZmultiprocessingr   ZthreadedrZ   ra   r\   )r_   r   rn   r   rd   )r   re   $test_threaded_get_within_distributed  s    
r  c             c   sN   | j dg|jdV \}| V  tdV  |jdks:t|j|jksJtd S )Nrk   )r   g?r  )	r   r   r   r   r	   r   r\   rh   rt   )r_   rR   rS   rT   r`   rd   rd   re   test_lose_scattered_data  s
    
r  c             c   sl   | j dg|jdV \}| j|ddV  | V  tdV  |jdksJt|j|j	gd|j	dikshtd S )	Nrk   )r   r   )r   g?r  )r   memory)
r   r   rv  r   r   r	   r   r\   rq  rh   )r  rR   rS   rT   r_   r`   rd   rd   re   "test_partially_lose_scattered_data  s    
r  c          	   c   s   | j ddddgg|jdV \}| jtd|jd}| jt||dd}tdV  | V  t	t
 t|V  W d Q R X |jd	kst|jd
kst|jd	kstd S )Nrk   r   rl   rm   )r   g?)r   g?r  r  )r   r   rZ   r<   r9   r   r	   r   r   r   r   r#   r   r\   )r_   rR   rS   rT   r`   rb   rc   rd   rd   re   test_scatter_compute_lose  s    
r  c             c   sj  | j d|jdV }| jt||jd}| td}| jt||d|jd}t|V  | V  t }x,|jdkrt	
dV  t |d k sbtqbW |jdkst|jdkst| t|}	t|	V  |j}
~t }x:|j|
gd|
d	ikrt	
dV  t |d k stqW |j}~t }xL|j|jkrd|
|jkrd||jkrdt	
dV  t |d k stqW d
S )a  
    Create irreplaceable data on one machine,
    cause a dependent computation to occur on another and complete

    Kill the machine with the irreplaceable data.  What happens to the complete
    result?  How about after it GCs and tries to come back?
    rk   )r   g?)r   r   r  g{Gz?r   )r   releasedN)r   r   rZ   r<   r9   r#   r   r,   r   r   r	   r\   rh   rq  rt   )r_   rR   rS   rT   r`   r   rb   rc   r   r  ZzkeyZxxkeyrd   rd   re   test_scatter_compute_store_lose  s8    	


r  c             c   s   | j dg|jdV \}| jt|dd}| t|}tdV  | V  t }x,|j	dkrztdV  t |d k sPt
qPW |j	d	kst
|j	d	kst
d
S )a  
    Create irreplaceable data on one machine,
    cause a dependent computation to occur on another and complete

    Kill the machine with the irreplaceable data.  What happens to the complete
    result?  How about after it GCs and tries to come back?
    rk   )r   g?)r   g?r  g{Gz?r   r  N)r   r   rZ   r8   r<   r   r	   r   r,   r   r\   )r_   rR   rS   rT   r`   rb   rc   r   rd   rd   re   *test_scatter_compute_store_lose_processing   s    	
r  c       	   	   c   s   t | j| jfddV }t | j| jfddV }|dd }|V }t|J tt|}|j|ksht	t
|j|jks|t	|V }||kst	W d Q R X | V  | V  d S )NT)r   c               S   s   dS )Nrk   rd   rd   rd   rd   re   r     s    z'test_serialize_future.<locals>.<lambda>)r!   r   r   rZ   r)   pickleloadsr6  rL   r\   r4   rh   r   r   )	rR   rS   rT   r_   rn   r   ra   r  r  rd   rd   re   test_serialize_future  s    

r  c          	   c   s   t | j| jfddV }t | j| jfddV }t|$ t |ksDtt||ksTtW d Q R X t|$ t |ksvtt||kstW d Q R X | V  | V  d S )NT)r   )r!   r   r   r)   r'   r\   r   )rR   rS   rT   r_   rn   rd   rd   re   test_temp_client,  s    


r  c       	   
   #   s   dd t dD }tt|fdd|D }tt|}| j|| |g t| j|jt||jg||ji||g d}t|V  t fdd|D stj	|j
kst|j|j	hdd	 |D B kstd S )
Nc             S   s   g | ]}t t|qS rd   )r   r<   )ri   rl  rd   rd   re   r   @  s    z(test_persist_workers.<locals>.<listcomp>rm   c                s   g | ]}t t| qS rd   )r   r   )ri   rl  )r|   rd   re   r   B  s    )r   r  c             3   s   | ]}|j  jkV  qd S )N)rh   r  )ri   r  )rS   rd   re   rj   M  s    z'test_persist_workers.<locals>.<genexpr>c             S   s   h | ]
}|j qS rd   )rh   )ri   r  rd   rd   re   r   P  s    z'test_persist_workers.<locals>.<setcomp>)rq   r   ru   r   rX  r   r#   rs   r\   rh   r  r  )	r  rR   rS   rT   r_   rz   r{   Ztotal2rv  rd   )rS   r|   re   test_persist_workers=  s    

r  c       	      #   s   dd t dD }tt|  fdd|D }| j||  g t||j |jt||jgi| g d}t|V  x$|D ]}|j|j |jhks|t	q|W x$|D ]}|j|j |jhkst	qW |j j |jhkst	|j
 jhdd |D B kst	d S )Nc             S   s   g | ]}t t|qS rd   )r   r<   )ri   rl  rd   rd   re   r   U  s    z(test_compute_workers.<locals>.<listcomp>rm   c                s   g | ]}t t| qS rd   )r   r   )ri   rl  )r|   rd   re   r   W  s    )r   r  c             S   s   h | ]
}|j qS rd   )rh   )ri   r  rd   rd   re   r   f  s    z'test_compute_workers.<locals>.<setcomp>)rq   r   ru   r   rX  r   r#   r"  rh   r\   r  )	r  rR   rS   rT   r_   rz   r{   rv  r  rd   )r|   re   test_compute_workersS  s    



r  c       	      c   s   t d}t d}|jdddd }| |gdd}|V }t|tsLt|d	 d
 |dd k snt|d dks~td S )Nz
dask.arrayrH  rV   )rg   )r   rk   rN  )r`   rb   r`   r   rb   )r   r   r   r   r]   r8  r\   rs   )	r_   rR   rS   rT   r   rK  r`   r   ra   rd   rd   re   test_compute_nested_containersi  s    

"r  c                 s"  dd t dD } tt|   fdd| D }t|dd\}}|dd |D ksXt|r`tt|dgd	\}}|d
d |D kstt|dd |D kstt| did	\}}| jdgikst| jgkstt| fdid	\}}| jdgikst| jgkstd S )Nc             S   s   g | ]}t t|qS rd   )r   r<   )ri   rl  rd   rd   re   r   x  s    z)test_get_restrictions.<locals>.<listcomp>rm   c                s   g | ]}t t| qS rd   )r   r   )ri   rl  )r|   rd   re   r   z  s    z	127.0.0.1Fc             S   s   i | ]}d g|j qS )z	127.0.0.1)rh   )ri   r6  rd   rd   re   r  }  s    z)test_get_restrictions.<locals>.<dictcomp>Tc             S   s   i | ]}d g|j qS )z	127.0.0.1)rh   )ri   r6  rd   rd   re   r    s    c             S   s   h | ]
}|j qS rd   )rh   )ri   r6  rd   rd   re   r     s    z(test_get_restrictions.<locals>.<setcomp>)rq   r   ru   r!   Zget_restrictionsr\   r  rh   )rz   r{   Zr1Zlooserd   )r|   re   test_get_restrictionsw  s    r  c             c   sD   |  dgV \}|jtkst|  ddiV }|d jtks@td S )Nrk   r`   g      ?)r   r:  rO  r\   float)r_   rR   rS   rT   r   r6  rd   rd   re   test_scatter_type  s    r  c             c   s   | j dg|jdV \}|j|jgdV  |j|jdiks<t|j|j|jhiksTt|j|j|jhikslt|j|jks|td S )Nrk   )r   )	r   r   retire_workersr  rh   r\   r   rj  r   )r_   rR   rS   rT   r`   rd   rd   re   test_retire_workers_2  s    r  c             '   s   |  ttdV } jdd |d d D dV  | |V }|ttdksTtt jt j  krtdkszn tt	dd |D stt	 fd	d|D stx4 j
 D ]&\}}d
t|  k rdk sn tqW d S )Nr   c             S   s   g | ]
}|j qS rd   )r   )ri   r  rd   rd   re   r     s    z,test_retire_many_workers.<locals>.<listcomp>r   )r   rl   c             s   s   | ]}|  V  qd S )N)r[   )ri   r   rd   rd   re   rj     s    z+test_retire_many_workers.<locals>.<genexpr>c             3   s    | ]} j |j jd kV  qdS )r  N)rt   rh   r  )ri   r   )rR   rd   re   rj     s    r  r  )r   ry   rq   r  rv   r\   rr   rj  r   rs   r  )r_   rR   r   r   r   r  r   rd   )rR   re   test_retire_many_workers  s     &r  )z	127.0.0.1rl   c             #   s   d|j d j_d|jd< ddd}| jdd	d
dgg jgdV }| jdgjgdV }| j|dd	d
dg||d}t|V  t fdd|D d	kst	tfdd|D dkst	d S )Ni@B stealingg{Gz?rn   r   c             S   s   t d | S )Ng{Gz?)r	   )r`   rb   rc   rd   rd   re   rn     s    z6test_weight_occupancy_against_data_movement.<locals>.frk   r   rl   rm   )r   )rb   rc   c             3   s   | ]}|j  jkV  qd S )N)rh   r  )ri   rn   )rS   rd   re   rj     s    z>test_weight_occupancy_against_data_movement.<locals>.<genexpr>c             3   s   | ]}|j  jkV  qd S )N)rh   r  )ri   rn   )rT   rd   re   rj     s    )r   r   )

extensions_pccallback_timera  r   r   rp   r#   ru   r\   )r_   rR   rS   rT   rn   rb   rc   r   rd   )rS   rT   re   +test_weight_occupancy_against_data_movement  s    


r  )z	127.0.0.1rV   c             c   sr   d|j d< d|jd j_ddd}| jdgd	d
V }| j|td|d}t|V  t|j	dt|j	 ksnt
d S )Ng{Gz?rn   i@B r  r   c             S   s   t d | S )Ng{Gz?)r	   )r`   rb   rd   rd   re   rn     s    z*test_distribute_tasks_by_ncores.<locals>.frk   T)r  rX   )rb   r   )r   )ra  r  r  r  r   rp   rq   r#   rr   r  r\   )r_   rR   rS   rT   rn   rb   r   rd   rd   re   test_distribute_tasks_by_ncores  s    


r  )rL   r  c       
      #   s   t   fdd} fdd| jtddd}| jtdd	d}| jtd
ddd}| jtddd}|| || || t||||fV  || t }	x*t dk rt |	 dk rt	
dV  qW  dd ||||fD kstd S )Nc                s   |    d S )N)add_done_callback)r   )r   rd   re   rn     s    z!test_add_done_callback.<locals>.fc                s     | j| jf d S )N)r   rh   r   )r   )Srd   re   r     s    z!test_add_done_callback.<locals>.grk   r  )rh   rD  r  r   g333333?r  )r   rh   rl   r`   rm   g       @g{Gz?c             S   s   h | ]}|j |jfqS rd   )rh   r   )ri   rn   rd   rd   re   r     s    z)test_add_done_callback.<locals>.<setcomp>)r  rZ   r<   r?   r8   r  r#   r,   rr   r   r	   r\   )
r_   rR   rS   rT   rn   r  r  r  r`   trd   )r  r   re   test_add_done_callback  s     



r  c       	      C   s   t td}t t|}t t|}| |}| |}t|jt|jd ksTtt|j|j t	sjtt|jt|jk std S )Nrk   )
r   r<   r   normalize_collectionrr   r   r\   r]   rh   r"   )	r_   rR   rS   rT   r`   rb   rc   r  r  rd   rd   re   test_normalize_collection  s    

r  c             c   s   t d}|jddd}|d }| |}| }t|j}	| |}
|j|	ksTtt	|jt	|
jksltt
dd |
j D stx,|j D ]\}}|
j| j|jkstqW | |V }| |
V }||kstd S )Nz
dask.arrayrV   )rg   )r   rk   c             s   s   | ]}t |tV  qd S )N)r]   r"   )ri   r  rd   rd   re   rj     s    z7test_normalize_collection_dask_array.<locals>.<genexpr>)r   r   r   r   ru   r8  r   r  r\   rr   r[  r\  r  rh   r   )r_   rR   rS   rT   r   r`   rb   r  rc   Zzdskr  r   r  r  r  rd   rd   re   $test_normalize_collection_dask_array  s    



r  c             C   s\   t d}|jddd}| }t| |  }~| |}|  }||ksXtd S )Nz
dask.arrayi   i   )r   )	r   r   r   r   r#   ru   r   r  r\   )r_   r   r`   rb   ZsolZ
normalizedr   rd   rd   re   /test_normalize_collection_with_released_futures  s    

r  c          	   c   s   t d}|jddd}t|jdks*ttjj| jgd |j	t
d|jd}| |}t|V  t }| | }	|	V  t }
|
| dk stt }| |d }t|V  t }
|
| dk stW d Q R X d S )	Nz
dask.arrayrV   rg   )r   r   )optimizationsrk   )r   dtype)r   r   r   rr   r   r\   r  r  _optimize_insert_futures
map_blocksr8   r  r   r#   r,   r   ru   )r_   rR   rS   rT   r   r`   rb   r  r   r   endrc   rd   rd   re   test_auto_normalize_collection0  s"    



r  c          	   C   s   t d}|jddd}|jtd|jd}| |}t| tj	j
| jgd. t }|   t }|| dk sxtW d Q R X d S )Nz
dask.arrayrV   rg   )r   rk   )r   r  )r  )r   r   r   r  r8   r  r   r#   r   r  r  r  r,   ru   r   r\   )r_   r   r`   rb   r  r   r  rd   rd   re   #test_auto_normalize_collection_syncJ  s    

r  c             C   sX   xR| j D ]H\}}}}}|dkr|dkrx(| D ]\}}||kr0|dkr0tq0W qW d S )Nr  r  Zwaiting)r  r  r\   )ri  rh   r   r  Zrecommendationsr   r   r  rd   rd   re   assert_no_data_lossZ  s    r  c             #   sv  dd l }||_dd tdD }dd |D }dd t||D }tt|}| |}	d tdV  d	d |D }
d
d |D }dd |D }x|j	rt
dd |j D rhtdV  t fdd|j|
d D }t fdd|j|d D }t fdd|j|d D }||  kr>|ksDn t||d k sVt||d k stqW t| d S )Nr   c             S   s   g | ]}t t|d dqS )g{Gz?)r   )r   r8   )ri   rl  rd   rd   re   r   e  s    z0test_interleave_computations.<locals>.<listcomp>r   c             S   s   g | ]}t t|d dqS )g{Gz?)r   )r   r:   )ri   r`   rd   rd   re   r   f  s    c             S   s"   g | ]\}}t t||d dqS )g{Gz?)r   )r   r9   )ri   r`   rb   rd   rd   re   r   g  s    )r  r  g?c             S   s   g | ]
}|j qS rd   )rh   )ri   r`   rd   rd   re   r   q  s    c             S   s   g | ]
}|j qS rd   )rh   )ri   rb   rd   rd   re   r   r  s    c             S   s   g | ]
}|j qS rd   )rh   )ri   rc   rd   rd   re   r   s  s    c             s   s   | ]}|j V  qd S )N)r  )ri   r  rd   rd   re   rj   u  s    z/test_interleave_computations.<locals>.<genexpr>g?c             3   s   | ]}| kV  qd S )Nrd   )ri   r  )r[   rd   re   rj   w  s   )r   c             3   s   | ]}| kV  qd S )Nrd   )ri   r  )r[   rd   re   rj   y  s   c             3   s   | ]}| kV  qd S )Nrd   )ri   r  )r[   rd   re   rj   {  s   rV   )distributedr   rq   rb  r   ru   r   r   r	   rt   r[  r   r\  rq  r\   r  )r_   rR   rS   rT   r  xsyszsr|   r   x_keysy_keysz_keysx_doney_donez_donerd   )r[   re   test_interleave_computationsa  s0    
"r  zNow prefer first-in-first-outc             #   s@  | j ttddd}| j t|dd}| j t||dd}d dd |D }dd |D }dd |D }	x|jrtd	d
 |j D r:t	
dV  t fdd
|j|d D }
t fdd
|j|d D }t fdd
|j|	d D }|
|  kr|ksn t|
|d k s(t||d k sftqfW d S )Nr   g{Gz?)r   )r  r  c             S   s   g | ]
}|j qS rd   )rh   )ri   r`   rd   rd   re   r     s    z4test_interleave_computations_map.<locals>.<listcomp>c             S   s   g | ]
}|j qS rd   )rh   )ri   rb   rd   rd   re   r     s    c             S   s   g | ]
}|j qS rd   )rh   )ri   rc   rd   rd   re   r     s    c             s   s   | ]}|j V  qd S )N)r  )ri   r  rd   rd   re   rj     s    z3test_interleave_computations_map.<locals>.<genexpr>g?c             3   s   | ]}| kV  qd S )Nrd   )ri   r  )r[   rd   re   rj     s   )r   c             3   s   | ]}| kV  qd S )Nrd   )ri   r  )r[   rd   re   rj     s   c             3   s   | ]}| kV  qd S )Nrd   )ri   r  )r[   rd   re   rj     s   rV   )rp   r8   rq   r:   r9   rt   r[  r   r\  r   r	   ru   rq  r\   )r_   rR   rS   rT   r  r  r  r  r  r  r  r  r  rd   )r[   re    test_interleave_computations_map  s$    "r  c             c   s8   | j ddi|j|jgdV  d|jks4d|jks4td S )NrS   rV   )r   )r   r   r  r\   )r_   rR   rS   rT   rd   rd   re   test_scatter_dict_workers  s    r  c              c   s   t  } tddd}t| d}tdV  y|d W n tk
rL   d S X t }x(|j	s|tdV  t |d k sVt
qVW | V  | V  d S )	Nz127.0.0.1:57484T)r   )rO   rm   )z	127.0.0.1i  g?r   )r   rS  r!   r-   r   r	   r   EnvironmentErrorr,   Zscheduler_commr\   r   )rO   r_   rR   r   rd   rd   re   test_client_timeout  s    

r  c             c   s@   |  dddgV }ddd}| j||d}|V }|dks<td S )	Nrk   r   rl   c             S   s   t | S )N)ru   )r   rd   rd   re   rn     s    z"test_submit_list_kwargs.<locals>.f)r   r   )N)r   rZ   r\   )r_   rR   rS   rT   r   rn   r   ra   rd   rd   re   test_submit_list_kwargs  s
    
r  c             c   sZ   |  dddgV }d
dd}| j|td|d}| |V }|dd	 tdD ksVtd S )Nrk   r   rl   c             S   s   | t | S )N)ru   )rl  r   rd   rd   re   rn     s    ztest_map_list_kwargs.<locals>.frV   )r   c             S   s   g | ]}|d  qS )r   rd   )ri   rl  rd   rd   re   r     s    z(test_map_list_kwargs.<locals>.<listcomp>)N)r   rp   rq   rv   r\   )r_   rR   rS   rT   r   rn   r   rd   rd   re   test_map_list_kwargs  s
    
r  c       	      c   st   t  }| dV }| jt|dd}x|j|jkr>tdV  q$W |j}~x&tdD ]}|j	| sdt
tjV  qRW d S )Nrk   g      ?)r   g{Gz?rg   )r,   r   rZ   r8   rh   rt   r   r	   rq   r  r\   r   )	r_   rR   rS   rT   r   r`   rb   rh   rl  rd   rd   re   test_dont_clear_waiting_data  s    r  c       	   	   c   sl   |  tdd}t|V  |jdks&t| |V \}}}}|jdksHttt	 ||| W d Q R X d S )Nrk   r   r  r>   )
rZ   r>   r#   r   r\   _get_futures_errorr@  r   r   r   )	r_   rR   rS   rT   rn   functionr   kwargsdepsrd   rd   re   test_get_future_error_simple  s    
r  c             c   s   t tddd}t tddd}t td|dd}t td|dd}t t||dd}| |}	t|	V  |	jd	ksxt| |	V \}
}}}|
j	d
kst|d|j
fkstd S )Nr   x0)r  rk   y0r`   rb   totr  r>   )r   r=   r>   ru   r   r#   r   r\   r  r@  rh   )r_   rR   rS   rT   r  r  r`   rb   r  rn   r  r   r   r  rd   rd   re   test_get_futures_error  s    

r  c          	   c   s   t td}t td}t td|}t td|}t t||}| |}	|	jdksZt| |	V \}
}}|	jdkszt|
jdkst|dkstt	
t |
|| W d Q R X d S )Nr   rk   pendingr  r>   )rk   r   )r   r=   r>   ru   r   r   r\   _recreate_error_locallyr@  r   r   r   )r_   rR   rS   rT   r  r  r`   rb   r  rn   r  r   r   rd   rd   re   test_recreate_error_delayed   s    
r	  c          	   c   s   |  td}|  td}|  td|}|  td|}|  t||}| |}	|	jdksZt| |	V \}
}}|	jdkszt|
jdkst|dkstt	
t |
|| W d Q R X d S )Nr   rk   r  r  r>   )rk   r   )rZ   r=   r>   ru   r   r   r\   r  r@  r   r   r   )r_   rR   rS   rT   r  r  r`   rb   r  rn   r  r   r   rd   rd   re   test_recreate_error_futures  s    
r
  c          	   c   s$  t jddd}|dd }| }| |}| |V \}}}tt ||| W d Q R X t	d}dd l
}	|j|	ddd	d
ddgid
d}
dd }|
j|}| |}| |V \}}}tt ||| W d Q R X | |}| |V \}}}tt ||| W d Q R X d S )NrV   rm   )r   c             S   s   d|  S )Nrk   rd   )r`   rd   rd   re   r   *  s    z0test_recreate_error_collection.<locals>.<lambda>zdask.dataframer   rS   rk   r   rl   )Z	chunksizec             S   s   | dkrt | S )Nr   )r$  )r`   rd   rd   re   make_err6  s    z0test_recreate_error_collection.<locals>.make_err)r   rq   rp   r   r   r  r   r   r   r   rI  from_pandasrJ  rS   r$  )r_   rR   rS   rT   rn   r  r   r   ddrL  dfr  df2Zdf3rd   rd   re   test_recreate_error_collection'  s(    

"

r  c       
      c   sh   t d}t d |j|jdddd  }| }| |V \}}}	dt|	dd	ksdt
d S )
Nz
dask.arrayZscipy)rV   rV   rV   )r   rk   z0.,0.,0. r  )r   r   ZlinalginvZzerosru   r   r  r   replacer\   )
r_   rR   rS   rT   r   rc   r  r   r   r   rd   rd   re   test_recreate_error_arrayH  s    

r  c          	   C   s~   |  td}|  td}|  td|}|  td|}|  t||}| |}tt}| | W d Q R X |j	dkszt
d S )Nr   rk   r  )rZ   r=   r>   ru   r   r   r   r   recreate_error_locallyr   r\   )r_   r  r  r`   rb   r  rn   r  rd   rd   re   test_recreate_error_syncR  s    
r  c          	   C   s@   |  td}tt}| | W d Q R X dt|ks<td S )Nr   zNo errored futures passed)rZ   r=   r   r   r$  r  r   r\   )r_   rn   r  rd   rd   re   test_recreate_error_not_error_  s    r  c             c   s|   t |j|j|jhkst| j|jgddV  t |j|jhksDtt }x,|jdkrvtdV  t |d k sLtqLW d S )NT)r   Zclose_workersr  g{Gz?rg   )	r  r   r   r\   r  r,   r   r   r	   )r_   rR   rS   rT   r   rd   rd   re   test_retire_workersf  s    r  c               @   s   e Zd ZdS )MyExceptionN)r@  rA  rB  rd   rd   rd   re   r  r  s   r  c          	   c   s~   G dd dt }tt | t| }W d Q R X | ttd}| 	|V }|t
tttdksjt|jrv|jsztd S )Nc               @   s   e Zd Zdd ZdS )z'test_robust_unserializable.<locals>.Fooc             S   s
   t  d S )N)r  )r  rd   rd   re   r  y  s    z4test_robust_unserializable.<locals>.Foo.__getstate__N)r@  rA  rB  r  rd   rd   rd   re   Foox  s   r  rV   )rD  r   r   r  rZ   r
   rp   r<   rq   rv   ry   r\   r  )r_   rR   rS   rT   r  r   r   r   rd   rd   re   test_robust_unserializablev  s    r  c          	   c   s   G dd dt }| t| }tt |V  W d Q R X | ttd}| 	|V }|t
tttdkspt|jr||jstd S )Nc               @   s   e Zd Zdd Zdd ZdS )z)test_robust_undeserializable.<locals>.Fooc             S   s   dS )Nrk   rd   )r  rd   rd   re   r    s    z6test_robust_undeserializable.<locals>.Foo.__getstate__c             S   s   t dd S )NrD  )r  )r  r  rd   rd   re   r    s    z6test_robust_undeserializable.<locals>.Foo.__setstate__N)r@  rA  rB  r  r  rd   rd   rd   re   r    s   r  rV   )rD  rZ   r
   r   r   r  rp   r<   rq   rv   ry   r\   r  )r_   rR   rS   rT   r  r   r   r   rd   rd   re   test_robust_undeserializable  s    r  c       	   	   c   s   G dd dt }| | d}tt}|V  W d Q R X | ttd}| |V }|t	tttdkspt
|jr||jst
d S )Nc               @   s$   e Zd Zdd Zdd Zdd ZdS )z2test_robust_undeserializable_function.<locals>.Fooc             S   s   dS )Nrk   rd   )r  rd   rd   re   r    s    z?test_robust_undeserializable_function.<locals>.Foo.__getstate__c             S   s   t dd S )NrD  )r  )r  r  rd   rd   re   r    s    z?test_robust_undeserializable_function.<locals>.Foo.__setstate__c             W   s   dS )Nrk   rd   )r  r   rd   rd   re   __call__  s    z;test_robust_undeserializable_function.<locals>.Foo.__call__N)r@  rA  rB  r  r  r  rd   rd   rd   re   r    s   r  rk   rV   )rD  rZ   r   r   r  rp   r<   rq   rv   ry   r\   r  )	r_   rR   rS   rT   r  r   r  r   r   rd   rd   re   %test_robust_undeserializable_function  s    
r  c             #   s   | j tddd}dd l  fdd}zVt|  || t }x,t dshtdV  t |d	 k s>tq>W  j	d
ksxtW d  `	X t }x0t
|jdkrtdV  t |d	 k stqW t|j|jhkstt|j|jhkstd S )Nrk   g?)r   r   c                s
   d _ d S )NrN  )foo)r`   )r  rd   re   rn     s    ztest_fire_and_forget.<locals>.fr  g{Gz?r   rN  )rZ   r8   r  r   r,   hasattrr   r	   r\   r  rr   rt   r  r  rh   )r_   rR   rS   rT   r   rn   r   rd   )r  re   test_fire_and_forget  s"    r!  c             c   sR   t | tdd tdV  t }x(|jrLtdV  t |d k s&tq&W d S )Nrk   r   g?g{Gz?)r   rZ   r>   r   r	   r,   rt   r\   )r_   rR   rS   rT   r   rd   rd   re   test_fire_and_forget_err  s    r"  c          
   C   s   t td}t| ddd"}|jttddd}td W d Q R X td	 | }|	 
d
}t|dksttx6|D ].}|rzd|kszd|kszt|dhkszt|qzW W d Q R X d S )Nr  Frm   )rO   r  Zthreads_per_workeri  g{Gz?)r   g?g?r'  r   ZReconnectingZgarbage-)rF   r   r   r!   rp   r8   rq   r	   r   r*  r+  rr   r\   r  )rO   loggerr_   r   rv  linesr  rd   rd   re   test_quiet_client_close  s    
r&  c          
   C   sn   d}xdt |D ]X}ttd,}t| d}t|| d}|  |  W d Q R X | }d|kstqW d S )Nrg   ztornado.application)rO   r   )	rq   rF   r   r   r   r!   r   r   r\   )rO   Z
n_attemptsr   r$  r6   rL   rv  rd   rd   re   <test_quiet_client_close_when_cluster_is_closed_before_client  s    
r'  c             c   s   t | jddV }|td}t|V  |j| jks6t| V  t	 }x4|j| jksZ| j
rztdV  t	 |d k sHtqHW d S )NT)r   rk   g{Gz?rg   )r!   r   rZ   r<   r#   r  r  r\   r   r,   rt   r   r	   )rR   rS   rT   r_   r   r   rd   rd   re   
test_close  s    

r(  c          	      sV    fdd}ddl m} |d,}t||td}|rBt|sFt~W d Q R X d S )Nc                sh   t dd}x6tdD ]*} ttdd}|| td qW  t	|  t
t	|}| S )Nr  )maxlenr   r   gMbP?)r   rq   rZ   r<   r.  r/  r   r	   rv   ry   ru   ra   )r   r6  rl  r   r|   )r_   rd   re   rn     s    

ztest_threadsafe.<locals>.fr   )ThreadPoolExecutorrX   )concurrent.futuresr*  ry   rp   rq   rs   r\   )r_   rn   r*  r  r   rd   )r_   re   test_threadsafe
  s    

r,  c                s`   t d}|jddd  fdd}ddlm} |d	}t||td	}|rXt|s\t	d S )
Nz
dask.arrayr   )rV   )r   c                s>   d}x4t dD ](}| tdd   7 }td qW |S )Nr   rX   gMbP?)rq   r.  r/  ru   r   r	   )r   r|   rl  )r`   rd   re   rn   !  s
    ztest_threadsafe_get.<locals>.fr   )r*  r   )
r   r   r   r+  r*  ry   rp   rq   rs   r\   )r_   r   rn   r*  r  r   rd   )r`   re   test_threadsafe_get  s    
r-  c                sb   t d}|jddd fdd}ddlm} |d	}t||td	}|rZt|s^t	d S )
Nz
dask.arrayr   )rV   )r   c                sH   d}x>t dD ]2} tdd  }|| 7 }td qW |S )Nr   rX   gMbP?)rq   r   r.  r/  ru   ra   r	   )r   r|   rl  r   )r_   r`   rd   re   rn   3  s    z"test_threadsafe_compute.<locals>.fr   )r*  r   )
r   r   r   r+  r*  ry   rp   rq   rs   r\   )r_   r   rn   r*  r  r   rd   )r_   r`   re   test_threadsafe_compute.  s    
r.  c             C   sT   | j  dst|j  ds(t|j  ds<t|j  dsPtd S )NrL   r  ri  )r  r  r   r\   )r_   rR   rS   rT   rd   rd   re   test_identityA  s    r/  c             c   st   t  | kst| jstdd }dd l}| |_z:| |td}| |V }|ttt	tdksftW d |`X d S )Nc             S   s:   t  }|t| }dd l}|jr$t||jks2t| S )Nr   )r   rZ   r<   r  r   r\   
tmp_clientra   )r`   rL   r   r  rd   rd   re   rn   N  s    
ztest_get_client.<locals>.fr   rg   )
r   r\   r   r  r0  rp   rq   rv   ry   r<   )r_   rR   rS   rT   rn   r  r   r   rd   rd   re   test_get_clientI  s    
r1  c           	   C   sD   ddl m}  | d d = d}tjtd|d t  W d Q R X d S )Nr   )_global_workersz.No global client found and no address providedz^{}$)r   )Zdistributed.workerr2  r   r   r$  r  r   )r2  r  rd   rd   re   test_get_client_no_cluster`  s
    
r3  c             #   sR   t d  jddd } fdd}| ||}|V }|ttdksNtd S )Nz
dask.arrayrV   )rg   )r   c                s   t |  jst|   S )N)r]   r<  r\   ru   r   )r`   )r   rd   re   rn   p  s    z%test_serialize_collections.<locals>.f)r   r   r   r   rZ   ru   rq   r\   )r_   rR   rS   rT   r`   rn   r   ra   rd   )r   re   test_serialize_collectionsk  s    
r4  c             c   s$   dd }|  |V }|dks td S )Nc              S   s   t  } t  | td S )Nrk   )r   r   rZ   r<   ra   )rL   rd   rd   re   rn   {  s    ztest_secede_simple.<locals>.fr   )rZ   r\   )r_   rR   rS   rn   ra   rd   rd   re   test_secede_simpley  s    r5  c       	      c   s   t  }dd }| |td}t }x6tdd |D s\tdV  t  |d k s(tq(W t	|j
dt	|j
 k sztt	|j
dt	|j
 k st| |V }|ttttd	gd kstd S )
Nc             S   s>   t  }td t  |jttdddd}|t| }|S )Ng{Gz?rV   F)r   r   )	r   r	   r   rp   r8   rq   rZ   ru   ra   )r`   rL   r   r|   rd   rd   re   rn     s    ztest_secede_balances.<locals>.fr   c             s   s   | ]}|j d kV  qdS )r  N)r   )ri   rn   rd   rd   re   rj     s    z'test_secede_balances.<locals>.<genexpr>g{Gz?r  r   rV   )	threadingactive_countrp   rq   r,   rs   r   r	   r\   rr   r  rv   ru   r<   )	r_   rR   rS   rT   r  rn   r   r   r   rd   rd   re   test_secede_balances  s    r8  c             c   sL   dd }| j |dd}tdV  t|jdkrH|jd |jd ksHtd S )Nc              S   s   t  } | jtdddd d S )Nrk   g?r8   )r   rh   )r   rZ   r8   )rL   rd   rd   re   rn     s    z#test_sub_submit_priority.<locals>.frn   )rh   g?r   r8   )rZ   r   r	   rr   rt   Z
prioritiesr\   )r_   rR   rS   rT   rn   r   rd   rd   re   test_sub_submit_priority  s
    r9  c                s\   |  dd }| fdd||gD ks,t|  dd }| fdd||gD ksXtd S )Nc               S   s
   t  jjS )N)r   ri  r   rd   rd   rd   re   r     s    z&test_get_client_sync.<locals>.<lambda>c                s   i | ]} d  |d  qS )r   rd   )ri   r  )rR   rd   re   r    s    z(test_get_client_sync.<locals>.<dictcomp>c               S   s
   t  jjS )N)r   ri  r   rd   rd   rd   re   r     s    c                s   i | ]} d  |d  qS )r   rd   )ri   r  )rR   rd   re   r    s    )rB  r\   )r_   rR   rS   rT   r   rd   )rR   re   test_get_client_sync  s    r:  c             c   sr   t d}t d}ddlm} |ddddgi}|j|dd	 }| |V }	|	V }
| |
V }||| d S )
NrI  zdask.dataframer   )	assert_eqr`   rk   r   rl   )r   )	r   r   dask.dataframe.utilsr;  rJ  r  r   r   r   )r_   rR   rS   rT   rL  r  r;  r  ddfr   Zddf2r  rd   rd   re   %test_serialize_collections_of_futures  s    

r>  c                s   t d}t d}ddlm  |ddddgi}|j|dd	 }| |}| } |	 | |j
|jksvt|  fd
d|| std S )NrI  zdask.dataframer   )r;  r`   rk   r   rl   )r   c                s    |   |S )N)r   )r`   rb   )r;  rd   re   r     s    z<test_serialize_collections_of_futures_sync.<locals>.<lambda>)r   r   r<  r;  rJ  r  r   r   ra   r   r:  r\   rZ   )r_   rL  r  r  r=  r   ra   rd   )r;  re   *test_serialize_collections_of_futures_sync  s    


r?  {Gz?c                sr   |dkrt t d  nt |  dkr.dS t  t }|jt fddtdD d|d}|t|}|	 S )Nr.  r   rm   c                s   g | ]} | d  qS )rk   rd   )ri   rl  )r`   rd   re   r     s    z%_dynamic_workload.<locals>.<listcomp>F)r   r   )
r	   r.  r   r   rp   _dynamic_workloadrq   rZ   ru   ra   )r`   r   rL   r   r|   rd   )r`   re   rA    s    
rA  c             C   s(   | j td|d}|jdddks$td S )Nr   )r   r   )rU   4   )rZ   rA  ra   r\   )r_   r   r   rd   rd   re   _test_dynamic_workloads_sync  s    rC  c             C   s   t | dd d S )Ng{Gz?)r   )rC  )r_   rd   rd   re   test_dynamic_workloads_sync  s    rD  c             C   s   t | dd d S )Nr.  )r   )rC  )r_   rd   rd   re   "test_dynamic_workloads_sync_random  s    rE  c             c   sh   d}| j td|d}|V }t|jtks,tt|j|hks@t||jksX||jksXt|dksdtd S )Ns   inc-123rk   )rh   r   )	rZ   r<   r:  rh   bytesr\   r  rt   r  )r_   rR   rS   rT   rh   r   ra   rd   rd   re   test_bytes_keys  s    rG  c             c   sp   t d}d}| jtd|d}|V }t |j|ks4tt|j|hksHt||jks`||jks`t|dksltd S )Nr  zinc-123rk   )rh   r   )r:  rZ   r<   rh   r\   r  rt   r  )r_   rR   rS   rT   uni_typerh   r   ra   rd   rd   re   test_unicode_ascii_keys  s    rI  c             c   s   t d}d}| jtd|d}|V }t |j|ks4tt|j|hksHt||jks`||jks`t|dkslt| t|}|V }	|	dkst| ddiV }
|
d V }|dkstd S )	Nr  u	   inc-123μrk   )rh   r   rl   zdata-123rN  )	r:  rZ   r<   rh   r\   r  rt   r  r   )r_   rR   rS   rT   rH  rh   r   ra   r  r  Zfuture3Zresult3rd   rd   re   test_unicode_keys  s    
rJ  c                s,   t j fdd}t| |}|dks(td S )Nc              3   s2     dV }  t| } |V }t|d S )NrN  )r   rZ   r<   rv   r   ZReturn)r`   rb   rc   )r_   rd   re   rn     s    z7test_use_synchronous_client_in_async_context.<locals>.f|   )r   r  r2   r\   )rO   r_   rn   rc   rd   )r_   re   ,test_use_synchronous_client_in_async_context  s    
rL  c             C   s   | }t |dd ddn}tdJ}t||d2}|dd td}td	 |  td	 W d Q R X W d Q R X | }|r~tW d Q R X d S )
Nr   F)rO   Zscheduler_portr  silence_logszdistributed.comm)rO   c             S   s   | d S )Nrk   rd   )r`   rd   rd   re   r   /  s    z5test_quiet_quit_when_cluster_leaves.<locals>.<lambda>rV   g?)	r   rF   r!   rp   rq   r	   r   r   r\   )rP   rO   r6   siorL   r   r  rd   rd   re   #test_quiet_quit_when_cluster_leaves)  s    

rO  c          
   C   sN   t jdd"}t|d | d}W d Q R X W d Q R X tdd |D sJtd S )NT)recordr   )rO   c             s   s   | ]}d t |jkV  qdS )r!   N)r   message)ri   r`  rd   rd   re   rj   =  s    z%test_warn_executor.<locals>.<genexpr>)warningscatch_warningsr   r[  r\   )rO   rR   rS   rT   rP  r_   rd   rd   re   test_warn_executor8  s    rT  c             #   s   | j tddd}| j tddd tdV  |  | j jgdgV }t fdd|D sbt|d |d ksvt|d } j|j	kr|n|}t
||jgkstt
||j  jgkstd	t|kstd
t|kstd S )Nrk   g      ?)r   g?)r   c             3   s&   | ]}t t|  jgkV  qd S )N)ry   r   r\  rh   )ri   ra   )r   rd   re   rj   F  s    z)test_call_stack_future.<locals>.<genexpr>r   r8   r:   )rZ   r:   r8   r   r	   
call_stackrh   rs   r\   	executingry   r   r   )r_   rR   rS   rT   r`   r   ra   r  rd   )r   re   test_call_stack_future@  s    rW  c             c   sv   | j tddd}tdV  |  V }|jr0|n|}t||jgksHtt||j |j	gksbtdt
|ksrtd S )Nrk   g      ?)r   g?r8   )rZ   r8   r   r	   rU  rV  ry   r   r\   rh   r   )r_   rR   rS   rT   r   ra   r  rd   rd   re   test_call_stack_allP  s    
rX  c             c   s^   t d}|jjdddjtdd }x|jsD|jsDtdV  q*W | 	|V }|sZt
d S )Nz
dask.arrayr   )rV   )r   g      ?)r   gMbP?)r   r   r.  r  r8   r   rV  r   r	   rU  r\   )r_   rR   rS   rT   r   r`   ra   rd   rd   re   test_call_stack_collections[  s    
rY  c             c   s\   t d}|jjdddjtdd }x|jsD|jsDtdV  q*W | 	 V }|sXt
d S )Nz
dask.arrayr   )rV   )r   g      ?)r   gMbP?)r   r   r.  r  r8   r   rV  r   r	   rU  r\   )r_   rR   rS   rT   r   r`   ra   rd   rd   re   test_call_stack_collections_alle  s    

rZ  Zprofile_cycle_interval)rL   Zworker_kwargsc             c   s   | j ttdd|jd}t|V  | jt d t d dV }|d rJt| jdt dV }|d tdd	 |j	D |j
d  kst| jt d
 t dV }d|d   k r|d k sn ttdd	 |j	D rt| j|jdV }|d rtd S )NrV   g?)r   r   rX   )r   stopr  r   c             s   s   | ]\}}|d  V  qdS )r  Nrd   )ri   r   r  rd   rd   re   rj   x  s    ztest_profile.<locals>.<genexpr>g333333?c             s   s   | ]\}}|d  V  qdS )r  Nrd   )ri   r   r  rd   rd   re   rj   }  s    )r   )rp   r8   rq   r   r#   r   r,   r\   ru   Zprofile_historyZprofile_recentr[  )r_   rR   rS   rT   r   r`   rb   ra   rd   rd   re   test_profileo  s    
*"r\  c          	   c   s   | j ttdd|jd}| j ttdd|jd}t|| V  | dV }| dV }|  V }|d |d |d  ks|ttt	
d"}	| dV }
|
t kstW d Q R X |	 }|rtd S )	NrV   g?)r   r   r8   r:   r  r  zdoes-not-exist)rp   r8   rq   r   r:   r#   r   r\   rF   r   r   Zcreater   )r_   rR   rS   rT   r`   rb   ZxpZypr  r$  Zprofrv  rd   rd   re   test_profile_keys  s    
r]  c          	   c   sX   t d2}t| jddddV }d|jks,t| V  W d Q R X | }d|ksTtd S )Nzdistributed.schedulerTr  F)r   rg  rM  )rF   r!   r   r  r\   r   r   )rR   rS   rT   rN  rL   r  rd   rd   re   test_client_with_name  s    


r^  c             c   s2   |  td}t|V  t|j}|j| ks.td S )Nrk   )rZ   r<   r#   r"   rh   rL   r\   )r_   rR   rS   rT   r`   r   rd   rd   re   &test_future_defaults_to_default_client  s    

r_  c             c   st   |  td}t|V  t|jddV }t|j|}t }x,|jdkrdt	
dV  t |d k s:tq:W | V  d S )Nrk   T)r   r  g{Gz?)rZ   r<   r#   r!   r   r"   rh   r,   r   r   r	   r\   r   )r_   rR   rS   rT   r`   rL   r   r   rd   rd   re   test_future_auto_inform  s    
r`  c           	   C   s4   t  $} td| d}|jst|  W d Q R X d S )NT)r   rO   )rK   r!   r   r\   r   )rO   rL   rd   rd   re   $test_client_async_before_loop_starts  s    
ra  )rL   r   rU   r   c             #   s@    fdd |   d}|V }|dks*tt|jdks<td S )Nc                sJ   t   st| dk r| S t | d }t | d }|| }| S )Nr   rk   )r   Zget_current_taskr\   r   r   )r`   rS   rT   r_   )fibrd   re   rb    s    z test_nested_compute.<locals>.fibr      r  )rZ   r\   rr   r  )r_   rR   rS   rT   r   ra   rd   )rb  re   test_nested_compute  s
    	rd  c          	   c   s  |  ddV  | dV }|dks&t| td}|j}t|V  |  |dV  | |V }|dksht~x||jkrt	dV  qlW t
t | |V  W d Q R X | |d V }|d kst|  ddgdV  | dV }|ddikst|  ddgdV  | dV }|dddks"t| ddgV }|dks@t|  ddd	d
gdV  | dV }|d	d
diiddks~td S )Nr`   rk   rN  g{Gz?rS   rT   r   )rS   rT   r_   r6  )Zset_metadataZget_metadatar\   rZ   r<   rh   r#   rt   r   r	   r   r   r  )r_   rR   rS   rT   ra   r   rh   rd   rd   re   test_task_metadata  s6    
re  c       	      c   s   t | ttdV  | jddV }|s,tx|D ]\}}d|ks2tq2W | jddV }t| |j	|j	hksttx.|
 D ]"}x|D ]\}}d|kstqW q~W d S )Nrg   )r   zdistributed.schedulerzdistributed.worker)r#   rp   r<   rq   Zget_scheduler_logsr\   Zget_worker_logsr  r   r   r\  )	r_   rR   rS   rT   Zlogsr   r  Zw_logsr  rd   rd   re   	test_logs  s    rf  c             c   sT   t td}| |}|V }|dks(tt|j|jg  krJ|jgksPn td S )Nrk   r   )r   r<   r   r\   ry   rt   rh   )r_   rR   rS   rT   r`   r   ra   rd   rd   re   test_avoid_delayed_finalize
  s
    
rg  c          
   c   st   t jd| jiX td$}tddV }|jj| jks<tW d Q R X | }| j|ks\t|	 V  W d Q R X d S )Nzscheduler-addresszdistributed.clientT)r   )
r   r  r  r   rF   r!   ri  r\   r   r   )rR   rS   rT   rN  r_   r  rd   rd   re   test_config_scheduler_address  s    
rh  c       	   	   C   s   t jdd}| dd dd }W d Q R X t|d j}d|ksFtd	|ksRtd
|ks^td|ksjtd|ksvtt|dk stt jdd2}dd }x"tdD ]}| dd ||}qW W d Q R X t|dk std S )NT)rP  c             S   s   | d S )Nrk   rd   )r`   rd   rd   re   r   #  s    z8test_warn_when_submitting_large_values.<locals>.<lambda>   0i r   z2.00 MBZlargez...z'000z000'i  rV   c             S   s   | S )Nrd   )r`   rb   rd   rd   re   r   0  s    r   )rR  rS  rZ   r   rQ  r\   rr   rq   )	r_   rR   rS   rT   rP  r   r  r  rl  rd   rd   re   &test_warn_when_submitting_large_values   s     rj  c             c   sj   t | jdddV }| j|j j}t }x4| j|j j|krZtdV  t |d k s(tq(W |	 V  d S )NTrV   )r   Zheartbeat_intervalg?rg   )
r!   r   clientsr  Z	last_seenr,   r   r	   r\   r   )rR   rS   rT   r_   Zlastr   rd   rd   re   r  5  s    zcloudpickle Py27 issuec             c   s(   ddi}|  |jdV }|dks$td S )NrS   rk   )rZ   r   r\   )r_   rR   rS   rT   r6  ra   rd   rd   re   test_unhashable_functionC  s    rl  c          	   c   sV   t jddi2 t| jddV }tdd t| jD s>tW d Q R X |	 V  d S )Nzclient-namezhello-worldT)r   c             s   s   | ]}d |kV  qdS )zhello-worldNrd   )ri   rg  rd   rd   re   rj   O  s    z#test_client_name.<locals>.<genexpr>)
r   r  r  r!   r   r[  ry   rk  r\   r   )rR   rS   rT   r_   rd   rd   re   test_client_nameK  s    &rm  c          	   C   sl   t |d | d}|td dks*tW d Q R X t |d | d}|td dks^tW d Q R X d S )Nr   )rO   rk   r   rl   )r!   rZ   r<   ra   r\   )rO   rR   rS   rT   r_   rd   rd   re   #test_client_doesnt_close_given_loopT  s    "rn  c          	   c   sR   d| j d _ttd}| V  |  V  W d Q R X | }d|ksNtd S )NrV   zscheduler-infozdistributed.clientBrokenPipeError)	Z_periodic_callbacksZintervalrF   r   r   r   Z_update_scheduler_infor   r\   )r_   rR   r$  r  rd   rd   re   test_quiet_scheduler_loss[  s    
rp  USERzno USER env variablec             C   s   t d ddlm} tdd|iidl\}\}}t|d | dH}tjd	d
i, |	 }dt
jd  d }||ks|tW d Q R X W d Q R X W d Q R X d S )Nbokehr   )BokehSchedulerZservices)rr  iC0  )Zscheduler_kwargsr   )rO   zdistributed.dashboard.linkzhttp://foo-{USER}:{port}/statuszhttp://foo-rq  z:12355/status)r   r   Zdistributed.bokeh.schedulerrs  r6   r!   r   r  r  r   r  environr\   )rO   rs  rR   rS   rT   r_   r  linkrd   rd   re   "test_diagnostics_link_env_variablee  s    
rv  c           
   c   sr   t jddiX t } tddd}tttf |V  W d Q R X t }|	 V  ||  dk sdt
W d Q R X d S )Nz!distributed.comm.timeouts.connect10msz127.0.0.1:3755T)r   rk   )r   r  r  r,   r!   r   r   r   r  r   r\   )r   r_   r[  rd   rd   re   test_client_timeout_2q  s    
rx  c           
   c   s   dd l } dd l} | jd| jjfg}| j|}|d tj	
ddi@ tddd}tttf |V  W d Q R X |jdd	V  W d Q R X |  d S )
Nr   /i  z!distributed.comm.timeouts.connectrw  z127.0.0.1:8080T)r   )Zfast)Ztornado.webZtornado.httpserverZwebZApplicationZRequestHandlerZ
httpserverZ
HTTPServerZlistenr   r  r  r!   r   r   r   r  r   r[  )tornadoZapplicationZhttp_serverr_   rd   rd   re   test_client_active_bad_port  s    
r{  c                s   t   fdd}|  d S )Nc          	   3   sZ  dd l }t| jdddgdV }z(|tdV dks8t||jdV  |dV  t	t
 |tV }W d Q R X |d	d
 t}t|V  t	t
 |j| dV  W d Q R X |dd
 V }t| ddgkst|dd
 V }|dkstt	t
 |dd
 V  W d Q R X t	t
 |dd
 V  W d Q R X W d | V  X d S )Nr   Tr   msgpack)r   serializersrk   r   rg   c             S   s   | S )Nrd   )r`   rd   rd   re   r     s    z4test_turn_off_pickle.<locals>.test.<locals>.<lambda>)r~  c               S   s   dS )Nrk   rd   rd   rd   rd   re   r     s    c               S   s   dS )Nrk   rd   rd   rd   rd   re   r     s    c               S   s   t S )N)r<   rd   rd   rd   re   r     s    c               S   s   t S )N)r<   rd   rd   rd   re   r     s    )rH  r!   r   rZ   r<   r\   r   r   r   r   r   r#   rv   rB  ry   r\  rE  r   )rR   rS   rT   rK  r_   r   ra   )r~  rd   re   test  s,    
z"test_turn_off_pickle.<locals>.test)rB   )r~  r~  rd   )r~  re   test_turn_off_pickle  s    $r  c          	   c   sf   dd l }t| jdddgdgdV }z2||dV }tt |V }W d Q R X W d | V  X d S )Nr   Tr|  r  )r   r}  deserializersrg   )	rH  r!   r   r   r   r   r   r   r   )rR   rS   rT   rK  r_   r   ra   rd   rd   re   test_de_serialization  s    r  c          	   c   s`   dd l }t| jddgdV }z2||dV }tt |V }W d Q R X W d | V  X d S )Nr   Tr|  )r   r  rg   )	rH  r!   r   r   r   r   r   r   r   )rR   rS   rT   rK  r_   r   ra   rd   rd   re   test_de_serialization_none  s    r  c             c   s&   t | jddV }| V  |  d S )NT)r   )r!   r   r   r   )rR   rS   rT   r_   rd   rd   re   test_client_repr_closed  s    
r  c          	   C   s,   t | dd}|  |  W d Q R X d S )NF)rO   r  )r!   r   r   )rO   r_   rd   rd   re   test_client_repr_closed_sync  s    r  c             c   s   t tddd}t tddd}tjt| | }| ||g\}}t||gV  ||j ||j k |j	t
|j j|j	t
|j jk kstd S )Nrk   )rS   r   )r  r   )rS   rV   )r   r<   r   orderr   Z__dask_graph__r   r#   rh   rt   r4   r   r\   )r_   rR   r  r`   rb   r  ZfxZfyrd   rd   re   test_nested_prioritization  s    r  c             c   sX   dd }|  dV }| ||}~t|V  |jdks:ttdV  |jdksTtd S )Nc             S   s   t dd S )NZlol)r   )r`   rd   rd   re   bad_fn  s    z)test_scatter_error_cancel.<locals>.bad_fnrk   r  g?)r   rZ   r#   r   r\   r   r	   )r_   rR   rS   rT   r  r`   rb   rd   rd   re   test_scatter_error_cancel  s    
r  c              C   s*   t tj} t dk s&tt|  d S )Nr   )r8  r6  Z_activer7  r\   ry   r\  )Zactiverd   rd   re   test_no_threads_lingering  s    
r  c             c   sP   t | jdddV }|jst| V  t | jdddV }|jrBt| V  d S )NT)r   direct_to_workersF)r!   r   r  r\   r   )rR   rS   rT   r_   rd   rd   re   test_direct_async  s    


r  c             C   s(   | j r
tdd }| | s$td S )Nc               S   s   t  jS )N)r   r  rd   rd   rd   re   rn     s    ztest_direct_sync.<locals>.f)r  r\   rZ   ra   )r_   rn   rd   rd   re   test_direct_sync  s    
r  c          	   c   sf   t | jddV }t | jddV }|td}tt |t| W d Q R X | V  | V  d S )NT)r   rk   )r!   r   rZ   r<   r   r   r$  r   )rR   rS   rT   r  r|  r   rd   rd   re   test_mixing_clients  s    
r  c             c   s@   t tddd}t t|dd}| |}|V dks<td S )Nrk   )r`   rk   )r  )rb   rk   rl   )r   r   r<   r   r\   )r_   rR   rS   rT   r`   rb   r   rd   rd   re   test_tuple_keys+  s    
r  c             c   st   t d}|jd}| jdd td|d}x|jsDtdV  q0W t|jdksXt	t
d	d
 |jD spt	d S )NrH  i c             S   s   | | S )Nrd   )rS   rT   rd   rd   re   r   7  s    z0test_map_large_kwargs_in_graph.<locals>.<lambda>r   )rT   g{Gz?e   c             s   s   | ]}| d V  qdS )ZndarrayN)r   )ri   r   rd   rd   re   rj   <  s    z1test_map_large_kwargs_in_graph.<locals>.<genexpr>)r   r   r.  rp   rq   rt   r   r	   rr   r\   r[  )r_   rR   rS   rT   rK  r`   r   rd   rd   re   test_map_large_kwargs_in_graph3  s    
r  c          
   c   sv   dd }t jjdd, | |}tt |V  W d Q R X W d Q R X t jjdd | V  |V  W d Q R X d S )Nc               S   s   t jdstd S )Nr  )r   r  r   r\   rd   rd   rd   re   rn   A  s    ztest_retry.<locals>.fF)r  T)r   r  r  rZ   r   r   r\   retry)r_   rR   rS   rT   rn   r   rd   rd   re   
test_retry?  s    

r  c          	   c   s   dd }|  |}|  t|}tt |V  W d Q R X tjjddH | V  |V }|dksft	| V  | V  |V }|dkst	W d Q R X d S )Nc               S   s   t jdS )Nr  )r   r  r   rd   rd   rd   re   rn   P  s    z"test_retry_dependencies.<locals>.fr   )r  r  )
rZ   r<   r   r   r  r   r  r  r  r\   )r_   rR   rS   rT   rn   r`   rb   ra   rd   rd   re   test_retry_dependenciesN  s    



r  c          	   c   s   dd }| j tddd}| j ||dd}~tt |V  W d Q R X tjjdd" | V  |V }|d	ksrt	W d Q R X d S )
Nc             S   s   t jdd S )Nr  rk   )r   r  r   )r`   rd   rd   re   rn   f  s    z%test_released_dependencies.<locals>.frk   r`   )rh   rb   r   )r  r  )
rZ   r<   r   r   r  r   r  r  r  r\   )r_   rR   rS   rT   rn   r`   rb   ra   rd   rd   re   test_released_dependenciesd  s    
r  c          	   c   s   t d ddlm} | jttdddV  | jddV \}}t||sLt	t
d	$}| j|d
V  tj|stt	W d Q R X d S )Nzbokeh.plottingr   )ModelrV   g?)r   T)ZplotZhtml)filename)r   r   Zbokeh.modelr  rp   r8   rq   r   r]   r\   r5   r  r  r  )r_   rR   rS   rT   r  r  Zfigurer  rd   rd   re   test_profile_bokehv  s    

r  c             c   s   |  tdd}ttddft|dfddd}dd|d	d
f|dd
fd}| j|ddd}|V }|dkshtt|ddf|ddftd|fddd}	dd|	ddfd}
| j|
dddV }|dkstd S )Nrk   r   _0_1_2)r  _3r  )r  r  rS   rT   r_   )rS   rT   r_   r6  r6  F)r2   rW   )r  r  _4r  r  rn   )r  rn   r   r   r  )rZ   r   r   r   r\   )r_   rR   rS   rT   r   Zsubgraphr&  r  ra   Z	subgraph2Zdsk2rd   rd   re   )test_get_mix_futures_and_SubgraphCallable  s$    

r  c       	      c   s   t d}dd l}|dtddi}|j|dd }|dd	 }|d d
|d< |dd	 }|d d
|d< | 	|V }|
|d
std S )Nzdask.dataframer   r`   rk   rW   r   )r   c             S   s   | S )Nrd   )r`   rd   rd   re   r     s    zJtest_get_mix_futures_and_SubgraphCallable_dask_dataframe.<locals>.<lambda>Zf8c             S   s   | S )Nrd   )r`   rd   rd   re   r     s    )r   r   rI  rJ  rq   r  r   Zmap_partitionsZastyper   Zequalsr\   )	r_   rR   rS   rT   r  rL  r  r=  ra   rd   rd   re   8test_get_mix_futures_and_SubgraphCallable_dask_dataframe  s    
r  )rl   rg   )*)r@  (  Z
__future__r   r   r   operatorr   collectionsr   r+  r   r   r"  r   r  r  r.  rw   r6  r   r,   r	   r  rR  r   r  r   Ztoolzr
   r   r   r   r   r   r   r   rz  r   Ztornado.ioloopr   r   r   Zdask.optimizationr   Zdask.bagr   r   r  r   r   r   r   r   r   r   r   r   r   Zdistributed.commr    Zdistributed.clientr!   r"   r#   r$   r%   r&   r'   r(   r)   rq  r*   r+   Zdistributed.metricsZdistributed.schedulerr-   r.   Zdistributed.sizeofr/   Zdistributed.utilsr0   r1   r2   r3   r4   r5   Zdistributed.utils_testr6   r7   r8   r9   r:   r;   r<   r=   r>   r?   r@   rA   rB   rC   rD   rE   rF   rG   rH   rI   rJ   rK   rL   r_   rM   r|  rN   rO   rP   rQ   rR   rS   rT   rf   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r  r  r  r  r
  r  r  r  Zmarkr   r  r  r  r  Zskipifplatformr   r!  r#  r%  r'  r(  r)  r*  r,  r-  r0  r1  r2  r3  r7  r<  r=  r>  rG  rM  rP  rR  rT  rU  rV  rW  rY  r]  rc  rd  re  rf  rh  rk  ro  rt  rx  rz  r}  r  r  r  r  r  r  r  Zparametrizer  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  rD  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r   r  r  r  r  r	  r  r  r  r  r  r  r  r  rx   r  r!  r$  r%  r&  r,  r0  r4  r5  r8  r9  r;  r=  r>  r?  r@  rA  rC  rF  rG  rI  rJ  rM  rP  rT  rU  rX  r^  r_  rd  re  rf  ri  rk  rl  rm  rn  ro  rp  rr  rs  rt  ru  rw  rx  ry  rz  r}  r~  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  Zxfailr  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r	  r
  r  r  r  r  r  r   r  r  r  r  r!  r"  r&  r'  r(  r,  r-  r.  r/  r1  r3  r4  r5  r8  r9  r:  r>  r?  rA  rC  rD  rE  rG  rI  rJ  rL  rO  rT  rW  rX  rY  rZ  r\  r]  r^  r_  r`  ra  rd  re  rf  rg  rh  rj  rl  rm  rn  rp  rt  rv  rx  r{  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  Z!distributed.tests.py3_test_clientrd   rd   rd   re   <module>   sf  (0, `,-/		

			

	



	

		, 
 "	'3 
' 	

)	 	  ""&"
" &  $
  	"" &"  
":

(4,"1$"	"$!
"(("""
"
$$	"&		
* )	