B
    T\                 @   sv  d dl Z e dZd dlmZ d dlmZ d dlmZ d dl	Z	d dl	m
Z
mZmZ d dlmZ d dlmZ d d	lmZ d d
lmZmZ d dlmZmZmZmZ deekreeddZeeddZdd Zedddd Zdd Zdd Zdd Zdd Z edddd Z!dd  Z"e j#j$d!d"e j#%d#d$d%d&gd'd( Z&eddd)d* Z'd+d, Z(d-d. Z)d/d0 Z*d1d2 Z+dS )3    Ndistributed)partial)add)gen)persistdelayedcompute)get_named_args)Delayed)tmpdir)waitClient)gen_clusterincclusterloopshould_check_stateF)r   c              C   s   ddl m}  d S )Nr   )r   )Zdask.distributedr   )r    r   :lib/python3.7/site-packages/dask/tests/test_distributed.pytest_can_import_client   s    r   T)clientc       	      c   s   t td}t|\}t|V  |j|jks<|j|jks<tt td}t|d\}}t|V  |j|jks||j|jks|td S )N   
   )r   r   r   r   keydataAssertionError)	csabxZx2yZy2Zoner   r   r   test_persist   s    


r"   c          
   C   s^  t  L\}\}}t|d | d& tdd }|d }|d }t|dd|gd|dfddg}t|d d	 tsxtt|d d
 d tstt|d d tstddddgddddgf}t| |kstt||g|ddgdd}|d d |kst|d d |kst|d  dks*t|dd  ddgfksFtW d Q R X W d Q R X d S )Naddress)r   r         )r   r      r   r   r         )   r%   F)Ztraverser)   )r   r   r   r   
isinstancer
   r   r   )r   r   r   r   r   resultZsolZresr   r   r   test_persist_nested)   s      r,   c       
      C   s   t d}t d}|ddddgi}t \}\}}t|d | db}|||g}||}	|j|		 |j
||gd	d
 t t |ddg}	W d Q R X W d Q R X W d Q R X d S )Npandaszdask.dataframer    r   r%      r#   )r   r   )axis)pytestimportorskip	DataFramer   r   scatterfrom_delayedutils	assert_eqr   concatraises	TypeError)
r   pddddfr   r   r   r   futuresddfr   r   r   !test_futures_to_delayed_dataframe>   s    


 r?   c          
   C   sz   t d}dddg}t V\}\}}t|d | d2}|||g}||}t||| ksbtW d Q R X W d Q R X d S )Nzdask.bagr   r%   r.   r#   )r   )r0   r1   r   r   r3   r4   listr   )r   ZdbLr   r   r   r   r=   r   r   r   test_futures_to_delayed_bagL   s    


rB   c       	   
      s   t d ddlm} t d}|dt r\}\}}t|d | dN}|g} j fdd	|D dd
}||	 |jgdd
 W d Q R X W d Q R X d S )Nz
dask.arrayr   )r6   Znumpyr$   r#   )r   c                s    g | ]} j |jjd qS ))shapedtype)r4   rC   rD   ).0f)dar    r   r   
<listcomp>^   s   z1test_futures_to_delayed_array.<locals>.<listcomp>)r/   )
r0   r1   Zdask.array.utilsr6   Zaranger   r   r3   Zconcatenater   )	r   r6   Znpr   r   r   r   r=   Ar   )rG   r    r   test_futures_to_delayed_arrayV   s    


rJ   c          	   c   sn   t jjdd ttd }W d Q R X tdV  |jr@t	ttdjdd}tdV  |jrjt	d S )Nsync)	schedulerr   g{Gz?r%   )
daskZconfigsetr   r   r   r   ZsleepZtasksr   )r   r   r   r   r    r   r   r   &test_local_get_with_distributed_activec   s    
rO   c          
   C   sN   ddl m} t 2\}\}}tj|d | d |  W d Q R X W d Q R X d S )Nr%   )test_to_hdfr#   )r   )dataframe.io.tests.test_hdfrP   r   r   r   )r   rP   r   r   r   r   r   r   test_to_hdf_distributedo   s    rR   zHDF not multi-process safe)reasonnpartitionsr   r&   r   c          
   C   sR   ddl m} t 6\}\}}tj|d |d |d |  W d Q R X W d Q R X d S )Nr%   )test_to_hdf_schedulersr#   )r   )rQ   rU   r   r   r   )rT   r   rU   r   r   r   r   r   r   !test_to_hdf_scheduler_distributedv   s    rV   c       	      c   sb   t d}t d}|ddddgddddgd}|j|dd	}|d
d}| |V  d S )Nr-   zdask.dataframer   r%   r.   r&   r   )r    r!   )rT   r!   count)r0   r1   r2   Zfrom_pandasgroupbyZaggr   )	r   r   r   r   r:   r;   r<   r>   r+   r   r   r   test_serializable_groupby_agg   s    

 rY   c          
   C   s   t  \}\}}t|d | dn}tdtd }}tt||}tt||}tt||}	||	}
tt|
d}|jdddkstW d Q R X W d Q R X d S )	Nr#   )r   r   r%   r   zdask.distributed)rL      )r   r   r   r   r   r   r   )r   r   r   r   r   r    r!   ZxxZyyZxxyyZxxyy2Zxxyy3r   r   r   test_futures_in_graph   s    
r[   c             C   s   t d}t d |jj}t x\}\}}t|d | dT t B}|jddd}|| |	|}||| |j
|j
kstW d Q R X W d Q R X W d Q R X d S )Nz
dask.arrayzarrr#   )r   )r.   r.   )r   r   )chunks)r0   r1   r5   r6   r   r   r   Zzerosto_zarrZ	from_zarrr]   r   )r   rG   r6   r   r   r   dZa2r   r   r   test_zarr_distributed_roundtrip   s    




r`   c             C   s   t d}t d}t p\}\}}t|d | ddidF}t t0 d}|jd|d	}|j||d	}|| W d Q R X W d Q R X W d Q R X d S )
Nz
dask.arrayr\   r#   Zset_as_defaultT)r   Zclient_kwargs)r   r   )r.   r.   )r]   )	r0   r1   r   r   r8   RuntimeErrorZonesZ
zeros_liker^   )r   rG   r\   r   r   r   r   zr   r   r   #test_zarr_in_memory_distributed_err   s    


rc   c          
      sr   t  b\}\}}t|d | d>}tdd    j|ddksDt| fddsZtW d Q R X W d Q R X d S )Nr#   )r   c               S   s   dS )Nr   r   r   r   r   r   <lambda>   s    z.test_scheduler_equals_client.<locals>.<lambda>)rL   r   c                s   |   jS )N)Zstoryr   )Zdask_scheduler)r    r   r   rd      s    )r   r   r   r   r   Zrun_on_scheduler)r   r   r   r   r   r   )r    r   test_scheduler_equals_client   s
    re   ),r0   r1   r   	functoolsr   operatorr   Ztornador   rM   r   r   r   Zdask.compatibilityr	   Zdask.delayedr
   Z
dask.utilsr   Zdistributed.clientr   r   Zdistributed.utils_testr   r   r   r   r   r"   r,   r?   rB   rJ   rO   rR   ZmarkZxfailZparametrizerV   rY   r[   r`   rc   re   r   r   r   r   <module>   s:   

