B
    T\i                 @   sD  d dl Z d dlZd dlmZmZ d dlZd dlZd dlZd dlm	Z	 d dl
Z
d dl
mZ d dlmZmZmZmZmZmZmZmZmZmZmZmZmZ d dlmZ d dlmZmZmZ d dl m!Z!m"Z" d d	l#m$Z$m%Z%m&Z& d d
l'm(Z( dd Z)e*dZ+e)dZ,e)dZ-e)dZ.e)dZ/e)dZ0e)dZ1dddZ2dddZ3dd Z4dd Z5dd Z6ej78d d!d" Z9ej78d d#d$ Z:ej78d d%d& Z;ej78d d'd( Z<ej78d d)d* Z=ej78d d+d, Z>ej78d d-d. Z?ej78d d/d0 Z@ej78d d1d2 ZAd3d4 ZBd5d6 ZCej78d7d8d9 ZDej78d7d:d; ZEej78d7d<d= ZFej78d7d>d? ZGd@dA ZHdBdC ZIdDdE ZJej78d dFdG ZKdHdI ZLdJdK ZMdLdM ZNej78d dNdO ZOej7PdPddQdRdSdTdUdUg dVi deQeRgdWdX ZSej78d dYdZ ZTej78d[ej7Pd\d]d^d_ ZUd`da ZVyd dlWZWeWXdbdReRfgZYW n eZk
rd   dZWY nX dcdd Z[G dedf dfeZ\dgdh Z]ej78didjdk Z^ej78dldmdn Z_ej78dldodp Z`ej78dqdrds Zaej78dtdudv Zbej78dqdwdx Zcej78dqdydz Zdej78d{d|d} Zeej78dld~d Zfdd Zgej78dlej7j8ejhjdddd Ziej78dlej7j8ejhjdddd Zjdd Zkdd Zldd Zmdd Zndd Zodd Zpdd Zqdd Zrdd Zsej78d{dd Ztdd Zudd Zvdd Zwdd Zxdd Zydd Zzdd Z{ej7Pdddgdd Z|dS )    N)addmul)merge)delayed)computetokenizenormalize_tokennormalize_function	visualizepersistfunction_cacheis_dask_collectionDaskMethodsMixinoptimizeunpack_collectionsnamed_schedulersget_scheduler)Delayed)tmpdirtmpfileignoring)incdec)longunicodePY2)Profilerc          	   C   s   t t t| S Q R X d S )N)r   BaseExceptionpytestimportorskip)path r!   3lib/python3.7/site-packages/dask/tests/test_base.pyimport_or_none   s    
r#   toolzz
dask.arrayzdask.bagzdask.dataframenumpyzscipy.sparsepandas   c             C   s   d S )Nr!   )abcr!   r!   r"   f1(   s    r+      c             C   s   d S )Nr!   )r(   r)   r*   r!   r!   r"   f2,   s    r-   c             C   s   d S )Nr!   )r(   r!   r!   r"   f30   s    r.   c               C   sv  t tstt dd stt tjtddt tjtddksDtt tjtddt tjtddksltt tjtddt tjtddkstt tttt tttkstt tttt tttkstt ttt ttkstt ttt ttkstt tjtddt tjtddksHtt tjtddt tjtddksrtd S )Nc             S   s   | S )Nr!   )r(   r!   r!   r"   <lambda>8   s    z)test_normalize_function.<locals>.<lambda>r,   )r)      r'   )	r	   r-   AssertionErrortzpartialr+   Zcomposer.   Zcurryr!   r!   r!   r"   test_normalize_function4   s$     "r4   c              C   s   d} t t| ttfstd S )N)r'   r,   r0   )
isinstancer   strbytesr1   )r(   r!   r!   r"   test_tokenizeQ   s    r8   znot npc               C   s4   t tjddt tjddks0td S )Ni  i  )r   nprandomRandomStateZrandom_sampler1   r!   r!   r!   r"   .test_tokenize_numpy_array_consistent_on_valuesV   s    r<   c               C   s   t tjdjdd d S )N   Zi2)dtype)r   r9   r:   astyper!   r!   r!   r"   /test_tokenize_numpy_array_supports_uneven_sizes\   s    r@   c               C   s   t tjdd d d  d S )N   r,   )r   r9   r:   r!   r!   r!   r"   'test_tokenize_discontiguous_numpy_arraya   s    rB   c               C   s   t tjdgdd d S )Nz2000-01-01T12:00:00zM8[ns])r>   )r   r9   arrayr!   r!   r!   r"   test_tokenize_numpy_datetimef   s    rD   c               C   sh   t tjdddt tjdddks(tt tjdgddgdd t tjdgddgdd ksdtd S )Ng      ?Zf8)r>   )r'   r,   )r(   Zi4)r)   Zi8r   )r   r9   rC   r1   r!   r!   r!   r"   test_tokenize_numpy_scalark   s    (rE   c               C   s   t tjdddgtdt tjdddgtdks4tt tjdd dgtdt tjdd dgtdkshtt tjdddgtdt tjdddgtdksttrt tjtdd	d
gtdt tjtdd	d
gtdkstd S )Nr(   Zaaaaa)r>   )r'   r(   )r'   N)r'   rF   u   Rebeca Alónzutf-8)encoding)r   r9   rC   objectr1   r   r   r!   r!   r!   r"   )test_tokenize_numpy_array_on_object_dtyper   s    rI   c        
   	   C   s  t d.} td}t| | ttj| dd}W d Q R X t d.} td}t| | ttj| dd}W d Q R X ||kstt d} tjjdd}t| | tj| dd}tj| dd}t|dd d f }t|dd d f }t|dd	d d f }t|d d df }	t	t
||||	gd
ks6tt|t|ksLtt|dd d f t|dd d f ksztW d Q R X d S )Nz.npy   r)	mmap_mode)
   rM   )sizer   r'   r0      )r   r9   arangesaver   loadr1   r:   Znormallenset)
fnxyzZmmZmm2r(   r)   r*   dr!   r!   r"   test_tokenize_numpy_memmap   s*    




rZ   c           
   C   sx   t df} t dR}td}t| | t|| tj| dd}|| }t|t|ks`tW d Q R X W d Q R X d S )Nz.npyrJ   rK   )rL   )r   r9   rP   rQ   rR   r   r1   )Zfn1Zfn2rV   r(   r)   r!   r!   r"   &test_tokenize_numpy_memmap_no_filename   s    
r[   c              C   sN   t tjdkstt tjdks$ttdd dd} t | t | ksJtd S )NZ 02106e2c67daf452fb480d264e0dac21Z c99e52e912e4379882a9a4b387957a0bc             S   s   | d S )Nr'   r!   )rV   r!   r!   r"   r/      s    z6test_tokenize_numpy_ufunc_consistent.<locals>.<lambda>r'   )r   r9   Zsinr1   ZcosZ
frompyfunc)r   r!   r!   r"   $test_tokenize_numpy_ufunc_consistent   s    r\   c              C   s,   t jtttd} t| }d}||ks(td S )N)r*   )s   cdask.tests.test_base
f3
p0
.)s   cdask.tests.test_base
f2
p0
.))r*   s   cdask.tests.test_base
f1
p0
.)r2   r3   r.   r-   r+   r   r1   )fressolr!   r!   r"   1test_tokenize_partial_func_args_kwargs_consistent   s    r`   c              C   s8   x2dt dddtdddgD ]} t| | kstqW d S )Nr'   g?1r,   r0   )r   slicer   r1   )ir!   r!   r"   test_normalize_base   s     rd   znot pdc              C   s   t jdddgddd gddddgd} t jdddgddd gddddgd}t| t|ks`td|j_t| t|ks|tt dddgd	d
d	gd} t dddgd	d
d	gd}| jd| d< t| t|kst| jd|d< t| t|kstd S )Nr'   r,   r0   4Zasd)rV   rW   )indexfoor(   r)   categoryrX   )pd	DataFramer   r1   rf   namerW   r?   )r(   r)   r!   r!   r"   test_tokenize_pandas   s    &&rl   c              C   s2   t jdddgddd gddddgd} t|  d S )Nr'   r,   r0   re   u   asd)u   xu   y)rf   )ri   rj   r   )dfr!   r!   r"   $test_tokenize_pandas_invalid_unicode   s    &rn   c              C   s@   t jdddddgddddd gidddgd} t|  d S )N   öutf8r'   r,   r0   )rf   )ri   rj   encoder   )rm   r!   r!   r"   (test_tokenize_pandas_mixed_unicode_bytes   s    4rr   c              C   s2   G dd dt } tddd |  gi}t| d S )Nc               @   s   e Zd ZdS )z0test_tokenize_pandas_no_pickle.<locals>.NoPickleN)__name__
__module____qualname__r!   r!   r!   r"   NoPickle   s   rv   rV   rg   )rH   ri   rj   r   )rv   rm   r!   r!   r"   test_tokenize_pandas_no_pickle   s    rw   c               C   sp   t dddt dddkstt dt dddks4tt dddt dddksPtt dddt dddksltd S )NrJ   r'   )rV   r,   )rW   )r   r1   r!   r!   r!   r"   test_tokenize_kwargs   s    rx   c              C   s0   G dd dt } t| dt| dks,td S )Nc               @   s   e Zd Zdd Zdd ZdS )z$test_tokenize_same_repr.<locals>.Fooc             S   s
   || _ d S )N)rV   )selfrV   r!   r!   r"   __init__   s    z-test_tokenize_same_repr.<locals>.Foo.__init__c             S   s   dS )Nza foor!   )ry   r!   r!   r"   __repr__   s    z-test_tokenize_same_repr.<locals>.Foo.__repr__N)rs   rt   ru   rz   r{   r!   r!   r!   r"   Foo   s   r|   r'   r,   )rH   r   r1   )r|   r!   r!   r"   test_tokenize_same_repr   s    r}   c              C   sz   G dd dt } | d| d }}t|t|ks6tt|t|ksJtt|}t| dd  t|}||ksvtd S )Nc               @   s   e Zd Zdd Zdd ZdS )z!test_tokenize_method.<locals>.Fooc             S   s
   || _ d S )N)rV   )ry   rV   r!   r!   r"   rz      s    z*test_tokenize_method.<locals>.Foo.__init__c             S   s   | j S )N)rV   )ry   r!   r!   r"   __dask_tokenize__  s    z3test_tokenize_method.<locals>.Foo.__dask_tokenize__N)rs   rt   ru   rz   r~   r!   r!   r!   r"   r|      s   r|   r'   r,   c             S   s
   | j d S )Nr'   )rV   )ry   r!   r!   r"   r/   
  s    z&test_tokenize_method.<locals>.<lambda>)rH   r   r1   r   register)r|   r(   r)   ZbeforeZafterr!   r!   r"   test_tokenize_method   s    r   c              C   s~   t dgt dgkstt dgt dks.tt dgt dgksFttd} td}d|d< t | gt |gksztd S )Nr'   r,   )r'   i  r   i  )r   r1   r9   rP   )rV   rW   r!   r!   r"   test_tokenize_sequences  s    

r   c               C   s$   t dddt dddks td S )Nr'   rV   )rV   r'   )r   r1   r!   r!   r!   r"   test_tokenize_dict  s    r   c               C   s(   t ddddht ddddhks$td S )Nr'   r,   rV   )r'   rV   )r   r1   r!   r!   r!   r"   test_tokenize_set  s    r   c           	   C   sp   t t^ ddlm}  | ddg}| ddg}| ddg}t|t|ksNtt|t|ksbtW d Q R X d S )Nr   )OrderedDict)r(   r'   )r)   r,   )r   ImportErrorcollectionsr   r   r1   )r   r(   r)   r*   r!   r!   r"   test_tokenize_ordered_dict#  s    
r   c              C   s.   t jddt jgdd} t| t| ks*td S )Nrg   u   JoséO)r>   )r9   rC   nanr   r1   )r(   r!   r!   r"   $test_tokenize_object_array_with_nans.  s    r   rV   Tr(      ag      ?y              ?r!   c             C   s   t | t | kst| d S )N)r   r1   )rV   r!   r!   r"   test_tokenize_base_types4  s    r   c              C   s\   t jd} t | d}| }t|t|ks8td|d d< t|t|ksXtd S )Ni  d   r'   rM   )r9   r:   r;   Zasmatrixrandcopyr   r1   )rngr(   r)   r!   r!   r"   test_tokenize_numpy_matrix:  s    r   znot spcls_name)ZdiaZbsrcooZcscZcsrdokZlilc          	   C   s   t jd}td  tjdd|d| }W d Q R X | }t	|t	|ksTt
t|drnd|jd d< n| dkrd|d< ntt	|t	|kst
td 2 | d	}t d|jd d< || }W d Q R X t	|t	|kst
d S )
Ni  rM   i'  )Zrandom_statedatar'   r   )r0   r0   r   )r9   r:   r;   r   Zwarnsspr   Zasformatr   r   r1   hasattrr   
ValueErrorrP   row)r   r   r(   r)   r!   r!   r"    test_tokenize_dense_sparse_arrayE  s      

r   c              C   sb   G dd dt } tdd }t|s(ttdr4tt| i sDtt|  rRtt| r^td S )Nc               @   s   e Zd ZdddZdd ZdS )z0test_is_dask_collection.<locals>.DummyCollectionNc             S   s
   || _ d S )N)dask)ry   dskr!   r!   r"   rz   f  s    z9test_is_dask_collection.<locals>.DummyCollection.__init__c             S   s   | j S )N)r   )ry   r!   r!   r"   __dask_graph__i  s    z?test_is_dask_collection.<locals>.DummyCollection.__dask_graph__)N)rs   rt   ru   rz   r   r!   r!   r!   r"   DummyCollectione  s   
r   r'   r,   )rH   r   r   r1   )r   rV   r!   r!   r"   test_is_dask_collectiond  s    r   
ADataClassc        
      C   s6  t dd } | d }| d }dd }|| ||dd | ||gD }t| \}}t|dks^t|d	d
dg}|d	d
dd	d
dg}||kstt|ddi\}}t|dkst|||ksttddddi\}}|rt||ddddifkstdd }	t| |	df|	ddfgdd\}}|| ||	dfg d S )Nr'   rJ   r,   c             S   sT   | |d| | |ddd|ggddd|dfd| dd	hi|f}t d k	rPt| d
|d d< |S )Nr(   r)   r'   r,   r*   rM   rY   er0   )r(   r]   )dataclassesr   )r(   r)   r*   iteratortr!   r!   r"   build  s    z&test_unpack_collections.<locals>.buildc             s   s   | ]
}|V  qd S )Nr!   ).0rc   r!   r!   r"   	<genexpr>  s    z*test_unpack_collections.<locals>.<genexpr>r0   z~az~bz~ctraverseFr(   c              W   s   t dd S )NzShouldn't have been called)r   )argsr!   r!   r"   fail  s    z%test_unpack_collections.<locals>.fail)r   )r   r   rS   r1   )
r(   r)   r*   r   r   r   Zrepackresultr_   r   r!   r!   r"   test_unpack_collections|  s*    r   c               @   sT   e Zd ZdZeejjZdd Z	dd Z
dd Zdd	 Zd
d Zdd Zdd ZdS )Tuple)_dask_keysc             C   s   || _ || _d S )N)r   r   )ry   r   keysr!   r!   r"   rz     s    zTuple.__init__c             C   s*   t |tr&tt| j|j| j|j S tS )N)r5   r   r   r   r   NotImplemented)ry   otherr!   r!   r"   __add__  s    
zTuple.__add__c             C   s   | j S )N)r   )ry   r!   r!   r"   r     s    zTuple.__dask_graph__c             C   s   | j S )N)r   )ry   r!   r!   r"   __dask_keys__  s    zTuple.__dask_keys__c             C   s   | j S )N)r   )ry   r!   r!   r"   r~     s    zTuple.__dask_tokenize__c             C   s   t dfS )Nr!   )tuple)ry   r!   r!   r"   __dask_postcompute__  s    zTuple.__dask_postcompute__c             C   s   t | jffS )N)r   r   )ry   r!   r!   r"   __dask_postpersist__  s    zTuple.__dask_postpersist__N)rs   rt   ru   	__slots__staticmethodr   threadedgetZ__dask_scheduler__rz   r   r   r   r~   r   r   r!   r!   r!   r"   r     s   r   c           	   C   s  ddd} t ddft ddfd}||  t ddftd	fd
}||  t| ddg}t|ddg}t|d	dg}tt d|_W d Q R X t|st	t
|t
|kst	t
|t
|kst	| dkst	t|||gdddgfkst	|| | }| dkst	| }t|ts(t	|jttdtddksHt	| dksZt	t|||\}}	}
||	 |
 }|j|jkst	d S )Nr'   r,   )r(   r)   r(   r)   r*   )r*   rY   rO   r   )r   r]   rY   r]   )r'   r,   )r0   rO   )rJ      )r'   r,   r0   rO   rJ   r   Zabcdefr=   )r   updater   r   r   raisesAttributeErrorrg   r   r1   r   r   r   r   r5   r   dictziprange)r   Zdsk2Zdsk3rV   rW   rX   r   Zt2x2y2z2Zt3r!   r!   r"   test_custom_collection  s4    


  r   znot dbc           	      s.  ddl m}  tjtddd}ttd}ttd}|	|	|}g  |  fdd	d
 |j
ddd W d Q R X tdd  D dksttdd  D dkstg  |  fdd	d
 |j
dd W d Q R X tdd  D dksttdd  D dksttdd  D dks*td S )Nr   )Callbackr   rO   )npartitionsr'   r,   c                s
     | S )N)append)keyr   )r   r!   r"   r/      s    z%test_compute_no_opt.<locals>.<lambda>)Zpretaskzsingle-threadedF)	schedulerZoptimize_graphc             S   s   g | ]}d |d kr|qS )r   r   r!   )r   kr!   r!   r"   
<listcomp>  s    z'test_compute_no_opt.<locals>.<listcomp>c             S   s   g | ]}d |d kr|qS )r   r   r!   )r   r   r!   r!   r"   r     s    c                s
     | S )N)r   )r   r   )r   r!   r"   r/     s    )r   c             S   s   g | ]}d |d kr|qS )r   r   r!   )r   r   r!   r!   r"   r   
  s    rA   c             S   s   g | ]}d |d kr|qS )r   r   r!   )r   r   r!   r!   r"   r     s    c             S   s   g | ]}d |d kr|qS )zadd-from_sequence-mulr   r!   )r   r   r!   r!   r"   r     s    )Zdask.callbacksr   dbfrom_sequencer   r2   r3   r   r   mapr   rS   r1   )r   r)   Zadd1Zmul2or!   )r   r"   test_compute_no_opt  s     r   znot dac              C   sh   t dd} tj| dd}|d }|d }t||\}}t || d sPtt || d sdtd S )Nr   )rM   rM   )rJ   rJ   )chunksr'   r,   )r9   rP   reshapeda
from_arrayr   allcloser1   )arrdarrZdarr1Zdarr2out1out2r!   r!   r"   test_compute_array  s    r   c              C   s~   ddl m}  tdd}tj|dd}|d |jdd }| }| || t	|j
|j
sftt|j
|jksztd S )	Nr   )	assert_eqr   )rM   rM   )rJ   rJ   )r   r'   )Zaxis)dask.array.utilsr   r9   rP   r   r   r   Zmeanr   rT   r   issubsetr1   rS   r   )r   r   rV   rW   r!   r!   r"   test_persist_array  s    
r   znot ddc              C   s   t ddddgddddgd} tj| dd}|jd }|j|j }t||\}}t jj	|| jd  t jj	|| j| j  d S )Nr'   r,   r0   rO   rJ   )r(   r)   )r   )
ri   rj   ddfrom_pandasr(   r)   r   utiltestingassert_series_equal)rm   ddfZddf1Zddf2r   r   r!   r!   r"   test_compute_dataframe'  s     
r   znot dd or not dac              C   s   t dd} tj| ddd }tddddgd	d	ddgd
}tj|ddj	d }t
||\}}t || d sxttjj||j	d  d S )Nr   )rM   rM   )rJ   rJ   )r   r'   r,   r0   rO   rJ   )r(   r)   )r   )r9   rP   r   r   r   ri   rj   r   r   r(   r   r   r1   r   r   r   )r   r   rm   r   Zarr_outZdf_outr!   r!   r"   test_compute_array_dataframe2  s     r   c              C   s0   t jtjdddgd} tj| dd d S )N)r0   r'   ro   rp   )r   columnsrO   )r   )ri   rj   r9   r:   rq   r   r   )rm   r!   r!   r"   -test_compute_dataframe_valid_unicode_in_bytes=  s    
r   c              C   s*   t jtjddgd} tj| dd d S )N)r0   r'   u   )r   r   rO   )r   )ri   rj   r9   r:   r   r   )rm   r!   r!   r"   &test_compute_dataframe_invalid_unicodeF  s    

r   znot da or not dbc                 sr   t jdddtdddg tt fdd t dd	\} }t	| tds\t
|dddgksnt
d S )
NrJ   r,   )r   r'   r0   c                  s
   t  S )N)r   r!   )r)   rV   r!   r"   r/   U  s    z(test_compute_array_bag.<locals>.<lambda>zsingle-threaded)r   )r   rP   r   r   r   r   r   r   r9   r   r1   )xxbbr!   )r)   rV   r"   test_compute_array_bagP  s    r   c              C   sT   t jddd} d}t| |\}}||  k s4t||ks@ttddksPtd S )NrJ   r,   )r   rM   )rJ   )r   rP   r   allr1   )rV   rW   r   Zyyr!   r!   r"   test_compute_with_literal\  s    r   c              C   s   t dd } | d }| d }t| dd|gd|dfddddgddfksNtt| |g|dd	}|d
 d
 | kstt|d
 d |kst|d dkstd S )Nr'   rJ   r,   )r(   r)   r   r=   )rA   r,   F)r   r   rA   )r   r   r1   )r(   r)   r*   r^   r!   r!   r"   test_compute_nestedh  s    r   z'graphviz exception with Python -OO flag)reasonc           	   C   sR  t d t 6} tjddd}|jtj| dd tj	tj| dsRt
|jtj| dd tj	tj| dst
t|d	dtj| dd tj	tj| dst
d	td
dftd
d	fd}t||tj| dd tj	tj| dst
t|d
ddg}t|tj| dd tj	tj| dsDt
W d Q R X d S )NZgraphvizrJ   r,   )r   Zmydask)filenamez
mydask.pngz
mydask.pdfr'   r(   )r(   r)   r*   r)   r*   )r   r   r   r   rP   r
   osr    joinexistsr1   r   r   r   )rY   rV   r   r!   r!   r"   test_visualizeu  s    

r   c           
   C   sj   t d tjddd} tdd>}| jd|dd	 t|}| }W d Q R X d
|ks\tW d Q R X d S )Nzmatplotlib.pyplotrJ   r,   )r   dot)	extensionorderZRdBu)Zcolorr   Zcmapzcolor="#)	r   r   r   rP   r   r
   openreadr1   )rV   rU   r]   textr!   r!   r"   test_visualize_order  s    

r   c              C   sX   dd l } ddlm} |d}| jd }tt|dd|j |j}t|}d|ksTt	d S )Nr   )dedentz%
    def inc():
        return x
    __main__z<test>execs   cloudpickle)
systextwrapr   modulesr  compile__dict__r   r   r1   )r  r   Zdefnr   r]   r   r!   r!   r"   5test_use_cloudpickle_to_tokenize_functions_in__main__  s    
r  c             C   sD   t | } x6| D ].}| | d tkrtf| | dd   | |< qW | S )Nr   r'   )r   r   r   )r   r   r   r!   r!   r"   
inc_to_dec  s
    
r  c           	   C   s^   t td} |  dkstt jjtgd |  dks@tW d Q R X |  dksZtd S )Nr'   r,   )optimizationsr   )r   r   r   r   r1   configrT   r  )rV   r!   r!   r"   test_optimizations_keyword  s
    r  c           	   C   sH  t td} t t| }| | }t| ||d\}}}}|dksFtt|j }t|j |ksbtt|j |ksttt |||t | ||kstt j| ||tgd}t| ||tgd\}	}
}t |	|
||kstt jj	tgd t| ||\}}}W d Q R X x>t
|	|
|g|||gD ]$\}}t|j t|j kstqW d S )Nr'   )r	  )r   r   r   r   r1   r   r   r  r
  rT   r   )rV   rW   rX   r   r   r   Zconstantr   Zsolsx3Zy3Zz3Zx4Zy4Zz4r(   r)   r!   r!   r"   test_optimize  s     
  r  c              C   s4  t td} t t| }| | }t| dd|gd|df}|d d }|d d d }|d d }t|tsttt|tstt|tstt|j t|j   krt|j ksn tt| ddddgddfkstt| |g|d	d
}|d d | kst|d d |kst|d  dks0td S )Nr'   r,   )r(   r)   r   r(   r)   r0   )rJ   r,   F)r   rJ   )	r   r   r   r   r5   r   r1   r   r   )r(   r)   r*   r   Za2Zb2Zc2r^   r!   r!   r"   test_optimize_nested  s     , r  c              C   sb   d} t tjd| g}tt| }d|ks2tdddddd	d
g}x|D ]}||ksJtqJW dS )zI
    Startup time: `import dask` should not import too many modules.
    zYif 1:
        import dask
        import sys

        print(sorted(sys.modules))
        z-cr   z
dask.arrayzdask.dataframer%   r&   ZpartdZs3fsZdistributedN)
subprocessZcheck_outputr  
executablerT   evaldecoder1   )codeoutr  Z	blacklistmodr!   r!   r"   test_default_imports  s    	

r  c               C   s   t ddddkstd S )Nr'   r,   r0   )r'   r,   r0   )r   r1   r!   r!   r!   r"   test_persist_literals  s    r  c              C   s   t dd } | d }| d }t| dd|gd|df}t|d d tsLtt|d d d tsftt|d d ts|tt| dddd	gdd
fkstt| |g|dd}|d d | kst|d d |kst|d  dkstd S )Nr'   rJ   r,   )r(   r)   r   r(   r)   r   r=   )rA   r,   F)r   rA   )r   r   r5   r   r1   r   )r(   r)   r*   r   r^   r!   r!   r"   test_persist_nested
  s     r  c              C   sr   t d} t t| }t t|}t|\}t|ts8t|j|jksHtt|jdksZt|	 |	 ksntd S )Nr'   )
r   r   r   r5   r   r1   r   rS   r   r   )Zx1r   r  r   r!   r!   r"   test_persist_delayed  s    
r  c           	   C   s  t jdddd } tdddgt}tt t	| | W d Q R X t	| |dd\}}t
|t jsjtt
|tjszt|j| jkst|j|jkstt|j|j  krt| jk sn tt|j|j  krt|jk sn tt| |stt|t|kstd S )NrJ   r,   )r   r'   r0   zsingle-threaded)r   )r   rP   r   r   r   r   r   r   r   r   r5   ZArrayr1   ZBagrk   rS   r   r   r9   r   list)rV   r)   r   r   r!   r!   r"   test_persist_array_bag'  s    ((r  c              C   s@   xt dD ]} tdd  q
W dtt  k r6dk s<n td S )Ni  c             S   s   | S )Nr!   )rV   r!   r!   r"   r/   ?  s    z6test_normalize_function_limited_size.<locals>.<lambda>2   iX  )r   r	   rS   r   r1   )rc   r!   r!   r"   $test_normalize_function_limited_size=  s    r  c           	   C   s  t d} t d}| jddd}dd }dd	lm} ||d
 tdd
  tjj|d& ||d
 tdd d
 d  W d Q R X ||d
 tdd
  |j	ddd}tjj|d@ tj
|d
 |tdd\}}||tdd d
 d  W d Q R X d S )Nz
dask.arrayzdask.bagrM   )rJ   )r   c             S   s   dd |   D S )Nc             S   s   i | ]\}}t d |f|qS )r,   )r   )r   r   vr!   r!   r"   
<dictcomp>K  s    zBtest_optimize_globals.<locals>.optimize_double.<locals>.<dictcomp>)items)r   r   r!   r!   r"   optimize_doubleJ  s    z.test_optimize_globals.<locals>.optimize_doubler   )r   r'   )array_optimizer,   )r   zsingle-threaded)r   )r   r   onesr   r   r9   r   r
  rT   r   r   r   r   )r   r   rV   r!  r   r)   r   r   r!   r!   r"   test_optimize_globalsD  s    

*r$  c           	      sn   t d} | jddd}|d d dd d d d d   fd	d
}tjjd |d    W d Q R X d S )Nz
dask.arrayrM   )rJ   )r   	   r'   rA   r,   c                s   | t  jkstt| |S )N)r   r   r1   r   )r   r   )rW   r!   r"   my_getc  s    z"test_optimize_None.<locals>.my_get)r"  r   )r   r   r#  r   r
  rT   r   )r   rV   r&  r!   )rW   r"   test_optimize_None]  s    
"r'  c           	   C   s   dd } | t d< ttd}z| dks.t|jdddksBttjjdd | dksbtW d Q R X | dks|ttjjdd |jdddkstW d Q R X W d t d= X d S )	Nc             [   s   dggS )N{   r!   )r   r   kwargsr!   r!   r"   schedulel  s    z(test_scheduler_keyword.<locals>.schedulerg   r'   r,   )r   r(  threads)r   r   r   r   r1   r   r
  rT   )r*  rV   r!   r!   r"   test_scheduler_keywordk  s    "r,  c           	   C   sF   t td} tt}| jtjd W d Q R X dt|j	ksBt
d S )Nr'   )r   z
scheduler=)r   r   r   r   	TypeErrorr   r   r   r6   valuer1   )rV   infor!   r!   r"   test_raise_get_keyword  s    r0  c            	   C   s|   t  d kstt ddtjjks$tt ddtjjks:ttjjdd t ddtjjks`tW d Q R X t  d ksxtd S )Nr+  )r   sync)	r   r1   r   r   r   ZlocalZget_syncr
  rT   r!   r!   r!   r"   test_get_scheduler  s     r2  c                 s@   dg  fdd} t dd  j| ddks0t d s<td S )	NFc                s   d d< t | |S )NTr   )r   r   )r   r   r   r)  )calledr!   r"   r     s    z$test_callable_scheduler.<locals>.getc               S   s   dS )Nr'   r!   r!   r!   r!   r"   r/     s    z)test_callable_scheduler.<locals>.<lambda>)r   r'   r   )r   r   r1   )r   r!   )r3  r"   test_callable_scheduler  s    r4  r   r+  Z	processesc          
      s   t dd   fddtdD }d}tjj|d& t }t|d| i}W d Q R X W d Q R X d	d
 |jD }t||kst	d S )Nc             S   s   t d | S )Ng      ?)timeZsleep)rV   r!   r!   r"   r]     s    
z"test_num_workers_config.<locals>.fc                s   g | ]} |qS r!   r!   )r   rc   )r]   r!   r"   r     s    z+test_num_workers_config.<locals>.<listcomp>rJ   r0   )num_workersr   c             S   s   h | ]
}|j qS r!   )Z	worker_id)r   rc   r!   r!   r"   	<setcomp>  s    z*test_num_workers_config.<locals>.<setcomp>)
r   r   r   r
  rT   r   r   ZresultsrS   r1   )r   r(   r6  ZprofZworkersr!   )r]   r"   test_num_workers_config  s    "r8  )r'   )r'   r,   )}r   r   operatorr   r   r  r  r5  r$   r   r   r   Z	dask.baser   r   r   r	   r
   r   r   r   r   r   r   r   r   Zdask.delayedr   Z
dask.utilsr   r   r   Zdask.utils_testr   r   Zdask.compatibilityr   r   r   Zdask.diagnosticsr   r#   r   r2   r   r   r   r9   r   ri   r+   r-   r.   r4   r8   ZmarkZskipifr<   r@   rB   rD   rE   rI   rZ   r[   r\   r`   rd   rl   rn   rr   rw   rx   r}   r   r   r   r   r   r   Zparametrizer6   intr   r   r   r   r   Zmake_dataclassr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   flagsr   r   r  r  r  r  r  r  r  r  r  r  r  r$  r'  r,  r0  r2  r4  r8  r!   r!   r!   r"   <module>   s   <


	


2(	



		