B
    F.\/                 @   s  d dl mZmZmZ d dlZed ed d dlmZ d dl	m
Z d dlmZ d dlmZ d dlmZmZmZ d dlZd dlZd dlm  mZ ejdd	d
dgid ddgdejddddgidddgdejddddgidddgdgZdd Zedddd d! Zedd"d#d$ Zedd"d%d& Z ej!"d'ed(d) gd*d+ Z#d,d- Z$d.d/ Z%edd"d0d1 Z&d2d3 Z'edd"d4d5 Z(dS )6    )print_functiondivisionabsolute_importNnumpypandas)wait)gen_cluster)clientcluster_fixtureloopx         
      )index            (   2         	   <   F   P   c             C   sr   t | t |kstt| tjr.t| | n@t| tjrHt| | n&t| tj	rbt
| | n| |ksntd S )N)typeAssertionError
isinstancepd	DataFrametmassert_frame_equalZSeriesZassert_series_equalZIndexZassert_index_equal)ab r(   Alib/python3.7/site-packages/distributed/tests/test_collections.pyassert_equal   s    r*      T)Ztimeoutr	   c          	   c   s   t jtjdtjddtdd}tj|dd}| |}|j|jksRt	| 
|}|V }t||j
dd dd	 d
d	 dd	 dd	 dd	 dd	 dd	 dd	 g}	x:|	D ]2}
|
|j
dd}| 
|
|}|V }t|| qW d S )Ni  )r   y)r   r   )npartitionssync)	schedulerc             S   s
   | j  S )N)r   mean)dfr(   r(   r)   <lambda>3   s    z!test_dataframes.<locals>.<lambda>c             S   s
   | j  S )N)r,   std)r1   r(   r(   r)   r2   4   s    c             S   s   | j | j| j d S )N)z)Zassignr   r,   Zdrop_duplicates)r1   r(   r(   r)   r2   5   s    c             S   s   | j S )N)r   )r1   r(   r(   r)   r2   6   s    c             S   s   | j S )N)r   )r1   r(   r(   r)   r2   7   s    c             S   s
   | j  S )N)r   Zcumsum)r1   r(   r(   r)   r2   8   s    c             S   s   |  ddg S )Nr   r,   )groupbycount)r1   r(   r(   r)   r2   9   s    c             S   s   | j dd S )Nr   K   )loc)r1   r(   r(   r)   r2   :   s    )r"   r#   nprandomZarangeddfrom_pandaspersistZ	divisionsr    computer$   r%   r*   )csr&   r'   r1   ZldfZrdfremoteresultexprsflocalr(   r(   r)   test_dataframes#   s,    



rF   )r	   c             c   s   dd l m} dd tdD }dd tdD }| |V }| |V }tjdj}	||dd|	}
||d	d
|	}||dd|	}||d	d
|	}dd dd dd dd g}xF|D ]>}||
|jdd}| |||}|V }t	||kst
qW d S )Nr   c             S   s.   i | ]&}t d D ]}tjdd||fqqS )r   )r   r   r   )ranger9   r:   ).0ijr(   r(   r)   
<dictcomp>F   s    z0test__dask_array_collections.<locals>.<dictcomp>r   c             S   s.   i | ]&}t d D ]}tjdd||fqqS )r   )r   r   r,   )rG   r9   r:   )rH   rI   rJ   r(   r(   r)   rK   H   s    r   r   ))r   r   r   )r   r   r,   ))r   r   )r   r   r   c             S   s
   | j | S )N)T)r   r,   r(   r(   r)   r2   T   s    z.test__dask_array_collections.<locals>.<lambda>c             S   s   |   |   S )N)r0   )r   r,   r(   r(   r)   r2   U   s    c             S   s   |  |jddS )Nr   )axis)dotr3   )r   r,   r(   r(   r)   r2   V   s    c             S   s   | | j ddd d d f  S )Nr   )rM   )r0   )r   r,   r(   r(   r)   r2   W   s    r.   )r/   )Z
dask.arrayZarrayrG   Z_scatterr9   r:   ZdtypeZArrayr>   allr    )r?   r@   r&   r'   daZx_dskZy_dskZ	x_futuresZ	y_futuresZdtZx_localZy_localZx_remoteZy_remoterC   exprrE   rA   r(   r(   r)   test__dask_array_collectionsB   s&    

rR   c             C   s8   t jddd}|dd }tdd |jD r4td S )Nd   r   )r-   c             S   s   | d S )N   r(   )r   r(   r(   r)   r2   e   s    z0test_bag_groupby_tasks_default.<locals>.<lambda>c             s   s   | ]}d |d kV  qdS )partdr   Nr(   )rH   kr(   r(   r)   	<genexpr>f   s    z1test_bag_groupby_tasks_default.<locals>.<genexpr>)dbrG   r5   anydaskr    )r?   r@   r&   r'   Zb2r(   r(   r)   test_bag_groupby_tasks_defaultb   s    r[   r   c             C   s   d S )Nr(   )r   r(   r(   r)   r2   i   s    r2   c             C   sZ   t jjddtttddddd}||}| | |jdd	d
}||}t|sVt	d S )NZ2000Z2001)valuenameidZ2HZ1Mr   )ZfreqZpartition_freqZseedr]   Ztasks)Zshuffle)
r;   ZdemoZmake_timeseriesfloatstrintr=   Z	set_indexlenr    )r   r	   r1   Zdf2r(   r(   r)   test_dataframe_set_index_synci   s    



rc   c             C   s0   t jj }tj|dd}|jdd   d S )Nr   )r-   z
2000-01-17z
2000-01-24)r"   utiltestingmakeTimeDataFramer;   r<   r8   r>   )r	   r1   ddfr(   r(   r)   test_loc_syncw   s    rh   c             C   s2   t jj }tj|dd}|jd 	  d S )Nr   )r-   r   )
r"   rd   re   rf   r;   r<   AZrollingr0   r>   )r	   r1   rg   r(   r(   r)   test_rolling_sync}   s    rj   c             c   s8   t jj }tj|dd}| |jdd }|V  d S )Nr   )r-   z
2000-01-17z
2000-01-24)r"   rd   re   rf   r;   r<   r>   r8   )r?   r@   r&   r'   r1   rg   futurer(   r(   r)   test_loc   s    rl   c          	   C   s  t jj }|jd |d< |jd |d< tj|dd}xdd dd gD ]}|||	t
}|||j	t
td	}t||jd
d  tdd |jD rt|||j	t
}|||jj	t
dtfd	}t||jd
d  tdd |jD rHtqHW tttf" ||ddg j	t
td	 W d Q R X |ddg	t
}|ddgj	t
td	}t||jd
d  d S )Ng?ri   Br   )r-   c             S   s   dS )Nri   r(   )r   r(   r(   r)   r2      s    z.test_dataframe_groupby_tasks.<locals>.<lambda>c             S   s   | j S )N)ri   )r   r(   r(   r)   r2      s    )metar.   )r/   c             s   s   | ]}d |d kV  qdS )rU   r   Nr(   )rH   rV   r(   r(   r)   rW      s    z/test_dataframe_groupby_tasks.<locals>.<genexpr>c             s   s   | ]}d |d kV  qdS )rU   r   Nr(   )rH   rV   r(   r(   r)   rW      s    )r"   rd   re   rf   ri   rm   r;   r<   r5   Zapplyrb   ra   r*   r>   Z
sort_indexrY   rZ   r    pytestZraisesNotImplementedError
ValueError)r	   r1   rg   Zindr&   r'   r(   r(   r)   test_dataframe_groupby_tasks   s$    &rr   c             c   s`   t d}t d}|jjddd}d||dk < ||j}| |jddd d	 }|V  d S )
Nsparsez
dask.array)rS   r   )r   r   )Zchunksr   gffffff?)rM   r   )ro   importorskipr:   Z
map_blocksZCOOr>   sum)r?   r@   r&   r'   rs   rP   r   rk   r(   r(   r)   test_sparse_arrays   s    

rv   ))Z
__future__r   r   r   ro   rt   Zdask.dataframeZ	dataframer;   Zdask.bagZbagrX   Zdistributed.clientr   Zdistributed.utils_testr   r	   r
   r   r   r9   r   r"   Zpandas.util.testingrd   re   r$   r#   Zdfsr*   rF   rR   r[   ZmarkZparametrizerc   rh   rj   rl   rr   rv   r(   r(   r(   r)   <module>   s0   

  