B
    OK\&              
   @   sh  d dl mZmZmZ d dlmZ d dlmZ d dlZd dlm	Z	 d dl
mZ d dlmZ d dlmZ d d	lmZ d d
lmZmZmZmZ d dlmZmZmZmZmZmZ edg ddd Zeddddddiifddddddifgddd Zeddddddiifdddddiifgddd Zeddddddiifdddddiifgddd Z eddddddiifdddddiifgddd Z!eddddddiifdddddiifgddd  Z"eddddddiifdddddiifgdd!d" Z#eddddddiifddddd#iifgdd$d% Z$eddddddiifdddddiifgdd&d' Z%eddddddiifdddddiifgdd(d) Z&eddd*ddd+iifdd*dddiifgdd,d Zeddd*dd+ddifgdd-d. Z'eddd+dddiifgdd/d0 Z(ej)j*d1d2eddd+dddiifdd+dddiifgdd3d4 Z+edd5gdd6d7 Z,eddddddiifdddddiifgdd8d9 Z-ej)j*d:d2eddddddiifdddddiifgdd;d< Z.ej)j/d=d2eddddddiifdddddiifgdd>d? Z0ej)1d@ej2dej)j/dAd2dBej2dCej)j3edDd2dBgdEdF Z4dS )G    )print_functiondivisionabsolute_import)time)delayedN)gen)Worker)wait)WINDOWS)tokey)incgen_clusterslowincslowadd)clientcluster_fixtureloopsabT)r   Zncoresc             c   s   |j r
t|jrtt|j|j|jddid}t|j|j|jdddd}| | gV  |j|jd|jdi|jdidkst|j |jddi|jdddikst|	 V  |j|jdii dkst|j |jddiikst|	 V  d S )NGPU   )r   	resources   )r   ZDB)
worker_resourcesAssertionErrorr   r   ipportr   _startaddress_close)cr   r   r    r"   ?lib/python3.7/site-packages/distributed/tests/test_resources.pytest_resources   s    


r$   z	127.0.0.1r   r   A   )r%   Bc             c   s   | j tdddid}| j tdddid}| j tdddid}t|V  |j|jksVtt|V  |j|jkspt|j|jgd|jd	ikstt|j|j	|j
dd
id}| V  t|V  |j|jkst| V  d S )Nr   r%      )r   r   r'   C)keysz	no-worker
   )r   r   )submitr   r	   keydatar   Zget_task_statusr   r   r   r   r   r    )r!   r   r   r   xyzdr"   r"   r#   test_resource_submit(   s    



r3   r'   c             #   sH    fddt dD }t|V  t|jdks2tt|jdksDtd S )Nc                s    g | ]} j t|d didqS )r%   r   )r   )r,   r   ).0i)r!   r"   r#   
<listcomp>C   s    z4test_submit_many_non_overlapping.<locals>.<listcomp>r&   r   )ranger	   lenr.   r   )r!   r   r   r   futuresr"   )r!   r#    test_submit_many_non_overlapping@   s    
r:   c             c   sL   | j dg|jdV \}| jt|ddid}t|V  |j|j dksHtd S )Nr   )workersr%   )r   r   )_scatterr   r,   r   r	   r.   r-   r   )r!   r   r   r   r/   futurer"   r"   r#   	test_moveJ   s    
r>   c             #   sV   j dg jdV \fddtdD }t|V  t fdd|D sRtd S )Nr   )r;   c          	      s$   g | ]} j t|d diddqS )r%   r   g?)r   delay)r,   r   )r4   r5   )r!   r/   r"   r#   r6   Z   s   z(test_dont_work_steal.<locals>.<listcomp>r+   c             3   s   | ]}|j  jkV  qd S )N)r-   r.   )r4   f)r   r"   r#   	<genexpr>^   s    z'test_dont_work_steal.<locals>.<genexpr>)r<   r   r7   r	   allr   )r!   r   r   r   r9   r"   )r   r!   r/   r#   test_dont_work_stealU   s
    
rC   c             c   sL   | j ttdddid}t|V  t|jdd |D ks>t|jrHtd S )Nr+   r'   r   )r   c             S   s   h | ]
}|j qS r"   )r-   )r4   r@   r"   r"   r#   	<setcomp>f   s    ztest_map.<locals>.<setcomp>)mapr   r7   r	   setr.   r   )r!   r   r   r   r9   r"   r"   r#   test_mapa   s    
rG   c             c   sp   t td}t t|}| j||g|ddi|ddiid\}}t||gV  |j|jks\t|j|jksltd S )Nr   r%   r'   )r   )r   r   persistr	   r-   r.   r   )r!   r   r   r   r/   r0   xxyyr"   r"   r#   test_persistj   s    &rK      c       	      c   s   t td}t t|}| j||ddi|ddiid}t|V  |jsJtdd tddD }| j|ddid}t|V  t|jdkstd S )	Nr   r%   r'   )r   c             S   s   g | ]}t t|qS r"   )r   r   )r4   r5   r"   r"   r#   r6      s    z test_compute.<locals>.<listcomp>r+      )r   r   computer	   r.   r   r7   r8   )	r!   r   r   r   r/   r0   rJ   ZxsZxxsr"   r"   r#   test_computex   s    


rO   c             c   s>   t dft dfd}| j|ddddiiddV }|dks:td S )	Nr   r/   )r/   r0   r0   r%   F)r   syncr(   )r   getr   )r!   r   r   r   Zdskresultr"   r"   r#   test_get   s    rS   c             c   sv   t td}t t|}| j||g||fddiid\}}t||gV  |j|jksXt|j|jksht|jrrtd S )Nr   r%   )r   )r   r   rH   r	   r-   r.   r   )r!   r   r   r   r/   r0   rI   rJ   r"   r"   r#   test_persist_tuple   s    "rT      r   c             c   s   | j ttdddidd}xLt|jt|j dk rftdV  t|jdksRtt|jdkstqW t	|V  |j
|jkst|j
|jkstd S )Nd   r%   r   g{Gz?)r   r?   g{Gz?r   )rE   r   r7   r8   r.   r   sleep	executingr   r	   total_resourcesavailable_resources)r!   r   r   r   r9   r"   r"   r#   r:      s    
c             c   sl   | j ttdddddd}x0t|jdk rLtdV  t|jdkstqW t	|V  |j
|jkshtd S )N   r   )r%   r'   g{Gz?)r   r?   g{Gz?)rE   r   r7   r8   r.   r   rW   rX   r   r	   rY   rZ   )r!   r   r   r9   r"   r"   r#   test_minimum_resource   s    
r\   c       	      c   s   | j ttddd}| j ttdddid}t }t|V  t }|| dk sRtt|j}t|j	}t
|t
|d	 k s~t|j	|j std S )
Ni  g?)r?   r+   r%   r   )r   rU   r   )rE   r   r7   r   r   r	   r   dicthas_what
processingr8   r   )	r!   r   r   r9   constrainedstartendr^   r_   r"   r"   r#   test_prefer_constrained   s    


rc    )reasonc             #   sr   | j ttdd jd}| j ttdddid}t|V  t fdd	|D sTttfd
d	|D sntd S )NrV   g?)r?   r;   r   r%   r   )r   c             3   s   | ]}|j  jkV  qd S )N)r-   r.   )r4   r@   )r   r"   r#   rA      s    z)test_balance_resources.<locals>.<genexpr>c             3   s   | ]}|j  jkV  qd S )N)r-   r.   )r4   r@   )r   r"   r#   rA      s    )rE   r   r7   r   r   r	   anyr   )r!   r   r   r   r9   r`   r"   )r   r   r#   test_balance_resources   s
    
rg   )z	127.0.0.1r   c             c   s   |j ddV  |jd dks t|jd dks2t|j|j ddiksJt| jtddddid}x|jd dkr~t	dV  qbW |j ddV  |jd dkst|jd dkst|j|j ddikstd S )Nr   )r%   r%   r   )r?   r   g{Gz?r(   )
set_resourcesrY   r   rZ   r   r   r,   r   r   rW   )r!   r   r   r=   r"   r"   r#   test_set_resources   s    ri   c             #   s   t d}|jddd}|dd }|dd }| }| j||gt| dd	iid
\}	}
t|	|
gV  t	 fdd| D st
d S )Nz
dask.arrayr+   )r&   )chunksc             S   s   | d S )Nr   r"   )r/   r"   r"   r#   <lambda>   s    z*test_persist_collections.<locals>.<lambda>c             S   s   d|  S )Nr   r"   )r/   r"   r"   r#   rk      s    r%   r   )r   c             3   s   | ]}t | jkV  qd S )N)r   r.   )r4   r-   )r   r"   r#   rA      s    z+test_persist_collections.<locals>.<genexpr>)pytestimportorskiparange
map_blockssumrH   tuple__dask_keys__r	   rB   r   )r!   r   r   r   dar/   r0   r1   wZwwrJ   r"   )r   r#   test_persist_collections   s    
&ru   z.Should protect resource keys from optimizationc       
      c   s   t d}|jddd}|dd }|dd }| }| j|t| dd	iid
V  x,tt	| D ]}	dt
||	ksltqlW d S )Nz
dask.arrayr+   )r&   )rj   c             S   s   | d S )Nr   r"   )r/   r"   r"   r#   rk      s    z(test_dont_optimize_out.<locals>.<lambda>c             S   s   d|  S )Nr   r"   )r/   r"   r"   r#   rk      s    r%   r   )r   rX   )rl   rm   rn   ro   rp   rN   rq   rr   rE   r   strZstoryr   )
r!   r   r   r   rs   r/   r0   r1   rt   r-   r"   r"   r#   test_dont_optimize_out   s    
 rw   z atop fusion seemed to break thisc             c   s`   t d}|jjddddd}|j|j }| j|t|jddiid	V  |j	sRt
|j	r\t
d S )
Nzdask.dataframeZ60sZ1dz
2000-01-01z
2000-01-31)ZfreqZpartition_freqra   rb   r%   r   )r   )rl   rm   ZdemoZmake_timeseriesr/   r0   rN   rq   dasklogr   )r!   r   r   r   ZddZdfr1   r"   r"   r#   test_full_collections	  s    



rz   optimize_graphz*don't track resources through optimization)ZmarksFzintermittent failurec       
      C   s   t d}dd }| j||d gd |jjdddd	 }|jt|jd
d	ii|d dd }| |}	|	|d  sxt|	|d  rtd S )Nz
dask.arrayc             S   s   | j f ddi d S )Nr%   r   )rh   )dask_workerr"   r"   r#   r@      s    ztest_collections_get.<locals>.fr   )r;   rV   )r+   )rj   r   r%   )r   r{   c             S   s
   t | jS )N)r8   ry   )r|   r"   r"   r#   g*  s    ztest_collections_get.<locals>.g)rl   rm   runZrandomrN   rq   rx   r   )
r   r{   r   r   r   rs   r@   r/   r}   Zlogsr"   r"   r#   test_collections_get  s    

r   )5Z
__future__r   r   r   r   rx   r   rl   Ztornador   Zdistributedr   Zdistributed.clientr	   Zdistributed.compatibilityr
   Zdistributed.utilsr   Zdistributed.utils_testr   r   r   r   r   r   r   r   r   r   r$   r3   r:   r>   rC   rG   rK   rO   rS   rT   r\   rc   Zmarkskiprg   ri   ru   rw   Zxfailrz   ZparametrizeZparamZskipifr   r"   r"   r"   r#   <module>   sd    " 	 
       &$"
 ""