ó
¦–Õ\c        	   @   sP  d  d l  Z  d  d l Z d  d l Z d  d l Z d  d l Z d  d l Z d  d l Z d  d l	 Z	 d  d l
 Z d  d l m Z d  d l j j Z d  d l Z d  d l j Z d  d l m Z d  d l m Z d  d l m Z m Z m Z m Z m Z m Z d  d l m  Z  m! Z! i e j" i d d d	 g d
 6d d d g d 6d d d d	 g ƒdk 6e j" i d d d g d
 6d d d g d 6d d d d g ƒdl 6e j" i d d d g d
 6d	 d d g d 6d d d d g ƒdm 6Z# e! i d d
 6d d 6d e j$ g  d ƒ ƒZ% e j" e# d e% d d d d g ƒ Z& e& j' ƒ  Z( e Z) e j* j+ d d d g ƒ d „  ƒ Z, d „  Z- d „  Z. e j* j+ d d d g ƒ d „  ƒ Z/ e j* j+ d d d g ƒ d „  ƒ Z0 e j* j+ d d d g ƒ d „  ƒ Z1 e j* j+ d d d g ƒ d  „  ƒ Z2 e j" i e j3 d d d	 g d	 d! d" ƒd# 6e j3 d d$ d% g d	 d! d& ƒd' 6e j5 d
 d d( g d	 ƒ j6 d) ƒ d* 6e j5 d+ d, d- g d	 ƒ d. 6e j3 e7 e8 e7 g d	 ƒ d/ 6e j5 e j9 d0 d1 d ƒƒ d2 6e j5 e j9 d0 d1 d d3 d4 ƒƒ d5 6e j5 e j: d6 d1 d ƒƒ d7 6ƒ Z; d8 „  Z< d9 „  Z= e j* j+ d: d d d e j> d; d< e j* j? ƒg ƒ d= „  ƒ Z@ e j* j+ d d d g ƒ d> „  ƒ ZA e j* j+ d d g ƒ d? „  ƒ ZB e j* j+ d d d g ƒ d@ „  ƒ ZC e j* j+ d d d g ƒ dA „  ƒ ZD e j* j+ d d d g ƒ dB „  ƒ ZE e j* j+ d d d g ƒ e j* j+ dC dD dE g ƒ dF „  ƒ ƒ ZF dG „  ZG dH „  ZH dI „  ZI dJ „  ZJ dK „  ZK dL „  ZL e j* j? e j* jM e jN dn k  dM dN ƒdO „  ƒ ƒ ZO dP „  ZP e j* j+ d d d g ƒ dQ „  ƒ ZQ dR „  ZR e j* j+ d d d g ƒ dS „  ƒ ZS e j* j+ d d d g ƒ dT „  ƒ ZT e j* j+ d d d g ƒ dU „  ƒ ZU dV „  ZV dW „  ZW dX „  ZX dY „  ZY dZ „  ZZ e j* j+ d[ e7 e8 g ƒ d\ „  ƒ Z[ d] „  Z\ d^ „  Z] d_ „  Z^ d` „  Z_ da „  Z` db „  Za dc „  Zb dd „  Zc de „  Zd df „  Ze e j* j? dg „  ƒ Zf e j* j+ d: d dh g ƒ di „  ƒ Zg dj „  Zh d S(o   iÿÿÿÿN(   t   copy(   t   delayed(   t   compute_as_if_collection(   t   shufflet   partitioning_indext   rearrange_by_columnt   rearrange_by_divisionst   maybe_buffered_partdt   remove_nans(   t	   assert_eqt	   make_metai   i   i   t   ai   i   t   bt   indexi    t   xi   i   i   i	   t   i8R   t   diskt   tasksc         C   sü   t  t t j d |  ƒ} t | t j ƒ s0 t ‚ | j t j k sH t ‚ t j	 | j | j
 d f ƒ } t j	 | j | j
 d f ƒ } t | j ƒ t | j ƒ @s§ t ‚ t | j ƒ j t j ƒ sÈ t ‚ t  t t j ƒ j
 t  t t j ƒ j
 k sø t ‚ d  S(   NR   i    i   (   t   shuffle_funct   dR   t
   isinstancet   ddt	   DataFramet   AssertionErrort   npartitionst   daskt   gett   _namet   sett
   issuperset(   R   t   sR   t   y(    (    s@   lib/python2.7/site-packages/dask/dataframe/tests/test_shuffle.pyt   test_shuffle(   s    #!c           C   s(   t  t t j ƒ j t j k s$ t ‚ d  S(   N(   R   R   R   R   R   (    (    (    s@   lib/python2.7/site-packages/dask/dataframe/tests/test_shuffle.pyt   test_default_partitions7   s    c       	   C   s1  t  j i t j j d ƒ d 6ƒ }  t j |  d d ƒ} t | | j d d d d d d	 ƒ} | j d
 d ƒ } | j	 d k s‚ t
 ‚ t | j ƒ j t | j ƒ ƒ s© t
 ‚ t | ƒ t |  ƒ k sÇ t
 ‚ t | j ƒ t |  j ƒ k së t
 ‚ t t t | j j ƒ  ƒ ƒ t t t |  j j ƒ  ƒ ƒ k s-t
 ‚ d  S(   Nid   R   R   i
   R   R   i   t
   max_branchi   t	   schedulert   sync(   t   pdR   t   npt   randomR   t   from_pandasR   R   t   computeR   R   R   R   R   t   lent   listt   columnst   mapt   tuplet   valuest   tolist(   t   dft   ddfR   t   sc(    (    s@   lib/python2.7/site-packages/dask/dataframe/tests/test_shuffle.pyt   test_shuffle_npartitions_task;   s    "$'$t   methodc         C   sB   d d l  m } | t t t j d |  ƒt t d d |  ƒƒ d  S(   Niÿÿÿÿ(   t   list_eqR   R   (   t   dask.dataframe.tests.test_multiR6   R   R   R   (   R5   R6   (    (    s@   lib/python2.7/site-packages/dask/dataframe/tests/test_shuffle.pyt   test_index_with_non_seriesI   s    c         C   s¿   t  t t d g d |  ƒj ƒ  } t  t d g d |  ƒj ƒ  } t  t d d |  ƒj ƒ  } t | j j ƒ  ƒ t | j j ƒ  ƒ k s‹ t ‚ t | j j ƒ  ƒ t | j j ƒ  ƒ k s» t ‚ d  S(   NR   R   (   R   R   R)   t   sortedR/   R0   R   (   R5   t   res1t   res2t   res3(    (    s@   lib/python2.7/site-packages/dask/dataframe/tests/test_shuffle.pyt   test_index_with_dataframeP   s
    "0c         C   s   t  j i d d d g d 6ƒ } t j | d ƒ } xe d d g D]W } t | d d | d |  ƒ} t | j d d ƒ ƒ t | j d d ƒ ƒ k s> t ‚ q> Wd  S(	   Ni   i   i   R   R   R   R#   R$   (   R%   R   R   R(   R   R*   R)   R   (   R5   R1   R   t   iR   (    (    s@   lib/python2.7/site-packages/dask/dataframe/tests/test_shuffle.pyt,   test_shuffle_from_one_partition_to_one_otherZ   s    c         C   s¡   t  j i d d d g d d 6ƒ } t j | d d ƒ} t | | j d d d |  ƒ} t t j | j | j ƒ  ƒ } x& | D] } | j	 | j	 k s{ t
 ‚ q{ Wd  S(	   Ni   i   i   i
   R   R   i   R   (   R%   R   R   R(   R   R   R   R   t   __dask_keys__R,   R   (   R5   R1   R2   R   t   partst   p(    (    s@   lib/python2.7/site-packages/dask/dataframe/tests/test_shuffle.pyt   test_shuffle_empty_partitionse   s    #t   dtypet   int32t   i32g      @g      @t   float32t   f32t   ct   categoryt   catR   t   et   ft   objt   boolt   20130101t   periodst   dtt   tzs
   US/Easternt   dt_tzt   2000t   tdc          C   su  t  t j d ƒ }  |  d k  |  d k @j ƒ  s4 t ‚ t t j |  ƒ ƒ d k sU t ‚ t  t j d ƒ t  t j d ƒ k j ƒ  s… t ‚ t  t d g d ƒ }  |  d k  |  d k @j ƒ  s½ t ‚ t t j |  ƒ ƒ d k sÞ t ‚ t  t d d d g d ƒ }  d |  k |  d k  @j ƒ  st ‚ t  t j d	 ƒ }  |  d	 k  |  d k @j ƒ  sPt ‚ t t j |  ƒ ƒ d k sqt ‚ d  S(
   Ni   i    i   RF   RK   RO   RH   i   i   (	   R   t   df2RF   t   allR   R*   R&   t   uniqueR   (   t   res(    (    s@   lib/python2.7/site-packages/dask/dataframe/tests/test_shuffle.pyt   test_partitioning_indexy   s    "!0"!""c          C   sú   t  j i t t j ƒ d 6d d d d g d d 6ƒ }  |  j j d ƒ |  _ |  j ƒ  } | j j j	 t t
 | j j j ƒ ƒ ƒ | _ t |  j d	 ƒ } t | j d	 ƒ } | | k j ƒ  sÀ t ‚ t |  d	 ƒ } t | d	 ƒ } | | k j ƒ  sö t ‚ d  S(
   NR   i   i   i   i   i   R   RJ   i   (   R%   R   R+   t   stringt   ascii_lettersR   t   astypeR    RK   t   set_categoriest   reversedt
   categoriesR   RX   R   (   R1   RW   RZ   R;   (    (    s@   lib/python2.7/site-packages/dask/dataframe/tests/test_shuffle.pyt-   test_partitioning_index_categorical_on_valuesŒ   s    -R   i   t   marksc         C   sp  t  j i t j j d ƒ d 6t j j d ƒ d d 6d t j j d ƒ ƒ} t j | d |  ƒ} t | j d ƒ | j d d d ƒƒ t | j d ƒ | j d d d ƒƒ t | j | j ƒ | j | j d d ƒƒ t | j | j | j	 ƒ | j | j | j	 d d ƒƒ t | j | j d	 ƒ | j | j d	 d d ƒƒ t | j | j
 ƒ | j | j
 d d ƒƒ d  S(
   Nid   R   gš™™™™™É?R   R   R   R   R   i   (   R%   R   R&   R'   R   R(   R	   t	   set_indexR   R   R   (   R   R1   R2   (    (    s@   lib/python2.7/site-packages/dask/dataframe/tests/test_shuffle.pyt   test_set_index_tasksœ   s      c         C   s§   t  j i t j j d ƒ d 6t j j d ƒ d d 6d t j j d ƒ ƒ} t j | d d ƒ} | j | j d |  ƒ} | | k sŠ t ‚ t	 | | j | j ƒ ƒ d  S(	   Nid   R   gš™™™™™É?R   R   R   i   R   (
   R%   R   R&   R'   R   R(   Rd   R   R   R	   (   R   R1   R   R   (    (    s@   lib/python2.7/site-packages/dask/dataframe/tests/test_shuffle.pyt   test_set_index_self_index·   s    c         C   s„  t  j i t j j d ƒ d 6t j j d ƒ d d 6d t j j d ƒ ƒ} t j | d d ƒ} t | j d d |  ƒj ƒ t | j d d |  ƒj ƒ k s¢ t	 ‚ t | j d d |  ƒj ƒ t | j d d |  ƒj ƒ k sä t	 ‚ t | j d d	 d d |  ƒj ƒ t | j d d	 d
 d |  ƒj ƒ k s2t	 ‚ t | j d d t
 d |  ƒj ƒ t | j d d t d |  ƒj ƒ k s€t	 ‚ d  S(   Nid   R   gš™™™™™É?R   R   R   i   R   R"   i   t   drop(   R%   R   R&   R'   R   R(   R   Rd   R   R   t   Truet   False(   R   R1   R2   (    (    s@   lib/python2.7/site-packages/dask/dataframe/tests/test_shuffle.pyt   test_set_index_namesÄ   s    ''!-!c      
   C   sq   t  j j d d i t d 6t d 6t d 6d d d d	 d
 d ƒ} | j d d |  ƒ} | j j ƒ  j	 d d ƒ d  S(   NRU   t   2004t   valuet   namet   idt   freqt   2Ht   partition_freqt   1Mt   seedi   R   R#   R$   (
   R   t   demot   make_timeseriest   floatt   strt   intRd   Rl   t   sumR)   (   R   R1   RW   (    (    s@   lib/python2.7/site-packages/dask/dataframe/tests/test_shuffle.pyt   test_set_index_tasks_2Ö   s
    	!c         C   s˜   t  j t j j d
 ƒ d d d g ƒ} t j | d d ƒ} | j d d |  d	 d d | j ƒ} | j d ƒ } t | | ƒ | j | j k s” t	 ‚ d  S(   Ni
   i   R,   R   R   R   i   R   R"   (   i
   i   (
   R%   R   R&   R'   R   R(   Rd   R   R	   R   (   R   R1   R2   t   ddf2RW   (    (    s@   lib/python2.7/site-packages/dask/dataframe/tests/test_shuffle.pyt   test_set_index_tasks_3à   s    'c         C   sŸ   t  j i d d d d d g d 6d d d d d g d	 6ƒ } t j | d
 d ƒ} | j d ƒ j ƒ  } | j d d |  ƒ} t | j d d !| j d d !ƒ d  S(   Ni   i   i   R   i	   i   i   i   R   R   R   (   R%   R   R   R(   Rd   t
   sort_indexR	   t   loc(   R   R1   R2   RW   R{   (    (    s@   lib/python2.7/site-packages/dask/dataframe/tests/test_shuffle.pyt   test_shuffle_sortì   s
    ;R#   t   threadst	   processesc   	         s)  t  j i t j j d ƒ d 6ƒ } t j | d d ƒ} | j d | j d ƒ } t | d d d d |  ƒ} | j	 | j	 k sƒ t
 ‚ t | j ƒ j | j ƒ s¤ t
 ‚ | j d	 | ƒ } t j j d	 | ƒ } | | j | j ƒ  ƒ } x? | j j ƒ  D]. ‰  t ‡  f d
 †  | Dƒ ƒ d k só t
 ‚ qó Wd  S(   Ni
   R   R   i   t   _partitionsR"   i    R   R#   c         3   s$   |  ] } ˆ  t  | j ƒ k Vq d  S(   N(   R   R‚   (   t   .0t   part(   R>   (    s@   lib/python2.7/site-packages/dask/dataframe/tests/test_shuffle.pys	   <genexpr>  s    i   (   R%   R   R&   R'   R   R(   t   assignR   R   R   R   R   R   t   issubsetR)   t   baset   get_schedulerR@   R‚   t   drop_duplicatesRy   (	   R   R#   R1   R2   R{   t   resultR   R   RA   (    (   R>   s@   lib/python2.7/site-packages/dask/dataframe/tests/test_shuffle.pyt   test_rearrange÷   s    "!c          C   s   d d l  m }  t j i d d d d d d g d	 6d d d d d d g d
 6ƒ } t j | d d d g ƒ } t | d	 d ƒ } |  | | ƒ d  S(   Niÿÿÿÿ(   R6   i   i   i   i   i   i   R   R   i    (   i    i   i   (   R7   R6   R%   R   R   t   repartitionR   (   R6   t   AR   R1   (    (    s@   lib/python2.7/site-packages/dask/dataframe/tests/test_shuffle.pyt.   test_rearrange_by_column_with_narrow_divisions  s
    Ac          C   s‰   d d  l  }  t ƒ  } | ƒ  } t | j  |  j ƒ s9 t ‚ t j t j | ƒ ƒ } | j sa t ‚ | ƒ  } t | j  |  j	 ƒ s… t ‚ d  S(   Niÿÿÿÿ(
   t   partdR   R   t   BufferR   t   picklet   loadst   dumpst   buffert   File(   R   RM   t   p1t   f2t   p2(    (    s@   lib/python2.7/site-packages/dask/dataframe/tests/test_shuffle.pyt   test_maybe_buffered_partd  s    			c          C   só   t  j i d d d d g d 6d d d d	 d
 g ƒ}  t j |  d d ƒ} d „  } t j j d | ƒ # | j d d d d d g ƒ} Wd  QX| j d k s¡ t	 ‚ |  j d ƒ } t
 | | ƒ t j t ƒ ! | j d d d d d g ƒWd  QXd  S(   Ni   i   i   i   R   R   i
   i   i   i(   R   c          _   s   t  ƒ  ‚ d  S(   N(   t	   Exception(   t   argst   kwargs(    (    s@   lib/python2.7/site-packages/dask/dataframe/tests/test_shuffle.pyt   throw$  s    R   t	   divisionsi   (   i   i   i   (   R%   R   R   R(   R   t   configR   Rd   Rž   R   R	   t   pytestt   raisest
   ValueError(   R1   R2   R   R{   RW   (    (    s@   lib/python2.7/site-packages/dask/dataframe/tests/test_shuffle.pyt&   test_set_index_with_explicit_divisions  s    4	$c          C   s¯   t  j i d d d d d d g d 6t d ƒ d	 6ƒ }  t j |  d ƒ } | j d	 d
 d d d g ƒ} | j d k sz t ‚ t | j d d ƒ j	 d ƒ d d g k s« t ‚ d  S(   Ni   i   i   i   i   i   R   t   abdabdR   Rž   R   RI   R   R#   R$   iþÿÿÿ(   R   RI   R   (
   R%   R   R+   R   R(   Rd   Rž   R   R)   R   (   R1   R2   RŠ   (    (    s@   lib/python2.7/site-packages/dask/dataframe/tests/test_shuffle.pyt   test_set_index_divisions_23  s
    5c          C   sZ  t  j d d d d d g d t ƒ}  t  j d d d d d g d t ƒ} t |  | ƒ t |  t j d ƒ ƒ t | t j d ƒ ƒ t |  j ƒ t | j ƒ k s¥ t ‚ t  j t  j	 d d d d g d t ƒ} t  j t  j	 d d d d g d t ƒ} t j
 ƒ  } | j	 | _ t | | ƒ t | | ƒ t | | ƒ t | j ƒ t | j ƒ k sVt ‚ d  S(   NR   Rž   i    i   i	   R)   (   R   Rd   Ri   Rh   R	   t   fullR*   R   R   R   R    R   (   t   d2t   d3t   d4t   d5t   exp(    (    s@   lib/python2.7/site-packages/dask/dataframe/tests/test_shuffle.pyt    test_set_index_divisions_compute=  s    $$$''c          C   s
  t  j i d d d g d 6d d d g d 6ƒ }  t  j i d d d	 g d 6d
 d
 d g d 6ƒ } t  j i d d d g d 6d d d g d 6ƒ } t j i |  d 6| d 6| d 6d |  d  d  d  d  g ƒ } | j ƒ  } d „  } t j j d | ƒ , | j d d d d d d g d t	 ƒ} Wd  QXt
 | | j d ƒ ƒ t j j d | ƒ , | j d d d d
 d d g d t	 ƒ} Wd  QXt
 | | j d ƒ ƒ t j t ƒ - | j d d d d
 d d d g d t	 ƒWd  QXt j t ƒ * | j d d d d
 d d g d t	 ƒWd  QXd  S(   Ni
   i   i   R   R   R   i   i   i   R   RI   i   i   i   R   RL   i    i   i   c          _   s   t  d ƒ ‚ d  S(   Ns   Shouldn't have computed(   Rš   (   R›   Rœ   (    (    s@   lib/python2.7/site-packages/dask/dataframe/tests/test_shuffle.pyR   Y  s    R   Rž   R9   (   R   i    (   R   i   (   R   i   (   R%   R   R   t   NoneR)   R   RŸ   R   Rd   Rh   R	   R    R¡   R¢   (   R–   R˜   t   p3R2   R1   R   RZ   (    (    s@   lib/python2.7/site-packages/dask/dataframe/tests/test_shuffle.pyt   test_set_index_divisions_sortedP  s"    ///	--.t   reasons&   multiprocessing spawn only after Py3.4c          C   sð   t  j i t j j d ƒ d 6t j j d ƒ d d 6d t j j d ƒ ƒ}  t j |  d d ƒ} | j ƒ  } t j d ƒ } | j	 d	 d
 ƒ } g  t
 d ƒ D] } | j t | d f ƒ ^ qš } t d „  | Dƒ ƒ } t | ƒ d k sì t ‚ d  S(   Nid   R   gš™™™™™É?R   R   R   i   t   spawnR   i   c         s   s   |  ] } | j  ƒ  Vq d  S(   N(   R   (   Rƒ   RŠ   (    (    s@   lib/python2.7/site-packages/dask/dataframe/tests/test_shuffle.pys	   <genexpr>{  s    i   (   R%   R   R&   R'   R   R(   t   clear_divisionst   mpt   get_contextt   Poolt   ranget   apply_asynct
   _set_indexR   R*   R   (   R1   R2   t   ctxt   poolt   _t   resultst   divisions_set(    (    s@   lib/python2.7/site-packages/dask/dataframe/tests/test_shuffle.pyt#   test_set_index_consistent_divisionsm  s    1c         O   s   |  j  | | Ž  j S(   N(   Rd   Rž   (   R1   R›   Rœ   (    (    s@   lib/python2.7/site-packages/dask/dataframe/tests/test_shuffle.pyR¸     s    c         C   sk   t  j i t j j d ƒ d 6ƒ } t j | d d ƒ} | j d d |  d d ƒ} | j d k  sg t ‚ d  S(   Nid   R   R   i2   R   t   autoi
   (	   R%   R   R&   R'   R   R(   Rd   R   R   (   R   R1   R2   R{   (    (    s@   lib/python2.7/site-packages/dask/dataframe/tests/test_shuffle.pyt'   test_set_index_reduces_partitions_smallƒ  s    "c         C   s3   t  j i t j j |  ƒ d 6t j j |  ƒ d 6ƒ S(   NR   R   (   R%   R   R&   R'   (   t   n(    (    s@   lib/python2.7/site-packages/dask/dataframe/tests/test_shuffle.pyt	   make_partŒ  s    c            s©   d } d } t  | | d ƒ ‰  t j ‡  f d †  t | ƒ Dƒ d t d ƒ d  g | d ƒ } | j d d |  d d	 d
 | ƒ} d | j k  oš d k  n s¥ t ‚ d  S(   Ng    €„.Ai2   i   c            s%   i  |  ] } t  ˆ  f d  | f “ q S(   R   (   RÂ   (   Rƒ   R>   (   RÁ   (    s@   lib/python2.7/site-packages/dask/dataframe/tests/test_shuffle.pys
   <dictcomp>–  s   	 R   i   R   R   R¿   t   partition_sizei   (	   Rx   R   R   R¶   RÂ   R­   Rd   R   R   (   R   t   nbytest   npartsR2   R{   (    (   RÁ   s@   lib/python2.7/site-packages/dask/dataframe/tests/test_shuffle.pyt'   test_set_index_reduces_partitions_large‘  s     	c            sœ   d } d } t  | | d ƒ ‰  t j ‡  f d †  t | ƒ Dƒ d t d ƒ d  g | d ƒ } | j d d |  d d	 d
 | ƒ} | j | j k s˜ t ‚ d  S(   Ni   g    €„.Ai   c            s%   i  |  ] } t  ˆ  f d  | f “ q S(   R   (   RÂ   (   Rƒ   R>   (   RÁ   (    s@   lib/python2.7/site-packages/dask/dataframe/tests/test_shuffle.pys
   <dictcomp>¢  s   	 R   i   R   R   R¿   RÃ   (	   Rx   R   R   R¶   RÂ   R­   Rd   R   R   (   R   RÅ   RÄ   R2   R{   (    (   RÁ   s@   lib/python2.7/site-packages/dask/dataframe/tests/test_shuffle.pyt)   test_set_index_doesnt_increase_partitions  s     	c         C   s…   t  j i t d ƒ d 6t d ƒ d 6ƒ } t j | d d d d d t ƒ} | j d d |  ƒ} t | j ƒ | j	 d	 k  s t
 ‚ d  S(
   Nid   R   R   R   i
   Rm   t   sortR   i   (   R%   R   R¶   R   R(   Ri   Rd   R*   R   R   R   (   R   R1   R2   R{   (    (    s@   lib/python2.7/site-packages/dask/dataframe/tests/test_shuffle.pyt"   test_set_index_detects_sorted_data©  s    )!c       ;   C   sž  t  j d d d d d d d d d d d d d d d	 d d d	 d
 d d d d d
 d d d d d d d d d d d d d d d d d d d d d d d d d d d d d d d d d d g: ƒ }  t j |  d d ƒ}  d d d  g } g  } xt t t | ƒ ƒ D]` } t | |  ƒ } t | | | d! !ƒ } | j t j i |  | | !d" 6d# t | | ƒ ƒƒ qú Wt	 j
 | ƒ j ƒ  } | j d" ƒ j j ƒ  j t k sšt ‚ d  S($   NI òñ07·I ìÞ4|
·I Ð‡£Á#·I žì:1·I ¤=2·I ò.]>·I °C·I Ì›R°C·I `ÑÉ°C·I XªûÆR·I V»BS·I  VÙBS·I ´Ë©¶Z·I žÔóOf·I ¦õ8d·I 8/ÌÕr·I žÑaÙ{·I Êä¦{·I F™<¥·I z_yw§·I ‡º(¹·I ^¥2¼·I jß}½·I R?LIÃ·I ´ßÀ|Ã·I ä»Ê‰Ã·I @½~¦Ö·t   unitt   nsi
   i$   i:   i   t	   timestampR   (   R&   t   arrayR%   t   to_datetimeR¶   R*   Ry   t   appendR   R   t   concatR²   Rd   R   R)   t   is_monotonicRh   R   (   t   valst   breakst   dfsR>   t   lot   hiR2   (    (    s@   lib/python2.7/site-packages/dask/dataframe/tests/test_shuffle.pyt   test_set_index_sorts²  s:    																		7c          C   sê  i t  j i d d d g d 6d d d g d 6d d	 d d g ƒd 6t  j i d d d g d 6d d d g d 6d d d d g ƒd 6t  j i d d d g d 6d d d g d 6d d d d g ƒd 6}  t j |  d
 t d	 d d d g ƒ } | j ƒ  } | j d d d ƒ} | j d k st ‚ | j j	 d k s5t ‚ t
 | | j d ƒ ƒ | j | j d d ƒ} | j d k sxt ‚ | j j	 d k st ‚ t
 | | j | j ƒ ƒ | j d ƒ } | j j	 d k sÐt ‚ t
 | | j d ƒ ƒ d  S(   Ni   i   i   R   i   i   R   R   i    R   i   i   i   i	   R   (   R   i    (   R   i   (   R   i   (   R%   R   R   t   metaR)   Rd   R   R   R   Rm   R	   R   (   t   dskR   R¦   R§   R¨   R©   (    (    s@   lib/python2.7/site-packages/dask/dataframe/tests/test_shuffle.pyt   test_set_indexÖ  s&    /,,$c          C   s*  t  j i d d d d d g d 6d d d d d g d 6ƒ }  t j |  d ƒ } | j d d d ƒ} | j d k sw t ‚ t | j ƒ t d d d d g ƒ k s¤ t ‚ | j d d d ƒ} | j d	 d k sÒ t ‚ d | j d k  o| j d k  od
 k  n st ‚ | j d d
 k s&t ‚ d  S(   Ni   i   i   R   g      ð?i   R   R   i    g       @(	   R%   R   R   R(   Rd   R   R   R   Rž   (   R1   R   t   d1R§   (    (    s@   lib/python2.7/site-packages/dask/dataframe/tests/test_shuffle.pyt   test_set_index_interpolateï  s    ;-;c          C   s†   t  t t d d d ƒ ƒ d ƒ }  t j i d |  d 6ƒ } t j | d ƒ } | j d d d ƒ} t d „  | j	 Dƒ ƒ s‚ t
 ‚ d  S(   Ni    iÈ   i
   i   R   R   c         s   s*   |  ]  } t  j t | ƒ t  j ƒ Vq d  S(   N(   R&   t
   issubdtypet   typet   integer(   Rƒ   R   (    (    s@   lib/python2.7/site-packages/dask/dataframe/tests/test_shuffle.pys	   <genexpr>  s    (   R9   R+   R¶   R%   R   R   R(   Rd   RX   Rž   R   (   t   LR1   R   RÛ   (    (    s@   lib/python2.7/site-packages/dask/dataframe/tests/test_shuffle.pyt   test_set_index_interpolate_intý  s
    "c    	      C   só  t  j t  j d d d ƒƒ }  t  j t  j d d d d d ƒƒ } t  j i | d 6|  d 6ƒ } t j | d ƒ } | j d d d ƒ} t  j |  j d	 |  j	 ƒ} | j
 d
 |  d
 k oÇ | d
 k n sÒ t ‚ | j
 d |  d k oø | d k n st ‚ | j d d d ƒ} t  j | d	 | j	 ƒ} | j
 d
 | d
 k sMt ‚ | j
 d | d k sjt ‚ | j
 d
 j | d
 j k st ‚ | j
 d
 j d  k	 s©t ‚ t  j | j d	 |  j	 ƒ} t j t ƒ  | j
 d
 | d
 k Wd  QXd  S(   NRP   RQ   i   RS   s
   US/Easternt   notzi   R   RD   i    iÿÿÿÿ(   R%   t   Seriest
   date_rangeR   R   R(   Rd   t   DatetimeIndexR/   RD   Rž   R   RS   R­   R    R¡   t	   TypeError(	   t   s_naivet   s_awareR1   R   RÛ   t   s1R§   t   s2t	   s2badtype(    (    s@   lib/python2.7/site-packages/dask/dataframe/tests/test_shuffle.pyt   test_set_index_timezone  s"    $11#Rg   c         C   s^  t  j i t d ƒ d 6d d d d d d d	 d
 d d g
 d 6d d d d d d d d d d g
 d 6ƒ } t j | d ƒ } t | j d d |  ƒ| j d d |  ƒƒ t | j d d |  ƒ| j d d |  ƒƒ t | j d d |  ƒ| j d d |  ƒƒ t | j | j d |  ƒ| j | j d |  ƒƒ t | j | j d |  ƒ| j | j d |  ƒƒ t | j | j	 d |  ƒ| j | j	 d |  ƒƒ t  j i t d ƒ d 6d d d d d d d	 d
 d d g
 d 6d d d d d d d d d d g
 d 6ƒ } t j | d ƒ } t | j d d |  ƒ| j d d |  ƒƒ t | j d d |  ƒ| j d d |  ƒƒ d  S(   Nt
   ABAABBABAAR   i   i   i   i   i   i   i   i   i	   i
   t   Bt   CRg   i    (
   R%   R   R+   R   R(   R	   Rd   R   Rî   Rï   (   Rg   t   pdfR2   (    (    s@   lib/python2.7/site-packages/dask/dataframe/tests/test_shuffle.pyt   test_set_index_drop  s0    %+%+c       	   C   s©   t  j i d d d d d d d g d 6d d d d d d d g d	 6ƒ }  t j |  d ƒ } d
 } t j t ƒ  } | j d d	 g ƒ Wd  QX| t | j	 ƒ k s¥ t
 ‚ d  S(   Ni   i   i   i   i   i   i   R   R   s1   Dask dataframe does not yet support multi-indexes(   R%   R   R   R(   R    R¡   t   NotImplementedErrorRd   Rw   Rl   R   (   R1   R2   t   msgt   err(    (    s@   lib/python2.7/site-packages/dask/dataframe/tests/test_shuffle.pyt(   test_set_index_raises_error_on_bad_input<  s    %"c       	   C   sÏ  t  j i d d d d g d 6d d d d g d	 6d d d d g d
 6ƒ }  t j |  d d t ƒ} | j sp t ‚ | j d d t ƒ} | j s” t ‚ t	 | j
 ƒ j t	 | j
 ƒ ƒ s» t ‚ xµ t t g D]§ } t | j d d | ƒ|  j d d | ƒƒ t | j | j d t d | ƒ|  j |  j d | ƒƒ t | j | j d d t d | ƒ|  j |  j d d | ƒƒ qÈ Wt j t ƒ  | j | j d t ƒWd  QXt j t ƒ  | j | j d t ƒWd  QXd  S(   Ni   i   i   i   R   i
   i   i(   R   t   zRÈ   R9   Rg   (   R%   R   R   R(   Ri   t   known_divisionsR   Rd   Rh   R   R   R†   R	   R   R    R¡   R¢   Rö   t   warnst   UserWarningR   (   R1   R   R   Rg   (    (    s@   lib/python2.7/site-packages/dask/dataframe/tests/test_shuffle.pyt   test_set_index_sorted_trueG  s&    '"!c          C   ss   t  j i d d d d g d 6d d d d g d 6ƒ }  t j |  d d ƒ} t | j d d	 t ƒ|  j d ƒ ƒ d  S(
   Ni   i   i   i   R   i    R   R   R9   (   R%   R   R   R(   R	   Rd   Rh   (   R1   R2   (    (    s@   lib/python2.7/site-packages/dask/dataframe/tests/test_shuffle.pyt&   test_set_index_sorted_single_partitiona  s    5c          C   sÏ   t  j i d d d g d 6d d d g d 6ƒ }  t  j i d d d g d 6d d d g d 6ƒ } t |  ƒ } t | ƒ } t j | | g d |  ƒ} | j s¡ t ‚ | j d d t ƒ} | j	 d	 k sË t ‚ d  S(
   Ni   i   i   R   i    R   RØ   R9   (   i    i   i   (
   R%   R   R   R   t   from_delayedR÷   R   Rd   Rh   Rž   (   R   R   t   aat   bbR1   RW   (    (    s@   lib/python2.7/site-packages/dask/dataframe/tests/test_shuffle.pyt"   test_set_index_sorted_min_max_sameh  s    //c             s  d d d g }  t  t t d „  g } xë | D]ã } t j g  |  D]  } i | | ƒ d 6| d 6^ q> d d d g ƒ} t j t j | d d ƒt j | | j | j j	 ƒ  k d d ƒg ƒ ‰  t
 ‡  f d	 †  t ˆ  j ƒ Dƒ ƒ sç t ‚ t ˆ  j d ƒ | j d ƒ ƒ s+ t ‚ q+ Wd  S(
   Ni   i   i   c         S   s   t  j |  d d ƒS(   NRÊ   RË   (   R%   RÎ   (   R   (    (    s@   lib/python2.7/site-packages/dask/dataframe/tests/test_shuffle.pyt   <lambda>}  t    R   R   R,   R   c         3   s'   |  ] } ˆ  j  | ƒ j ƒ  j Vq d  S(   N(   t   get_partitionR)   t   empty(   Rƒ   RB   (   R2   (    s@   lib/python2.7/site-packages/dask/dataframe/tests/test_shuffle.pys	   <genexpr>‡  s    (   Rx   Rv   Rw   R%   R   R   RÐ   R(   R   t   maxt   anyR¶   R   R   R	   Rd   (   t	   test_valst
   converterst   convR>   R1   (    (   R2   s@   lib/python2.7/site-packages/dask/dataframe/tests/test_shuffle.pyt   test_set_index_empty_partitionv  s    B1+c          C   s  d d d d g }  t  t t d „  g } xâ | D]Ú } t j g  |  D]  } i | | ƒ d 6| d 6^ qA ƒ } t j | d d ƒ} | j d k s” t ‚ | | j	 | j	 j
 ƒ  k j d ƒ } | | j	 | j	 j
 ƒ  k j d ƒ } t | | ƒ só t ‚ | j d k s. t ‚ q. Wd  S(	   Ni   i   i   i   c         S   s   t  j |  d d ƒS(   NRÊ   RË   (   R%   RÎ   (   R   (    (    s@   lib/python2.7/site-packages/dask/dataframe/tests/test_shuffle.pyR   ‘  R  R   R   R   (   Rx   Rv   Rw   R%   R   R   R(   R   R   R   R  Rd   R	   (   R  R  t	   converterR   R1   R2   t   expected_df(    (    s@   lib/python2.7/site-packages/dask/dataframe/tests/test_shuffle.pyt   test_set_index_on_empty‹  s    6%%c          C   s  d d l  m }  t j i d d d d g d 6d d	 d	 d
 g d 6d d d d g d 6d d d d d	 g ƒ} t j | d d t ƒ} | j s’ t ‚ |  | ƒ } t	 | ƒ } | | _
 t | | d t ƒ| j sÕ t ‚ t j | j d ƒ d d t ƒ} t j t ƒ  |  | ƒ Wd  QXd  S(   Niÿÿÿÿ(   t   compute_divisionsi   i   i   i   R   i
   i   i(   R   Rö   R   RÈ   t   check_divisions(   t   dask.dataframe.shuffleR  R%   R   R   R(   Ri   R÷   R   R    Rž   R	   Rd   R    Rø   Rù   (   R  R1   R   Rž   R   RI   (    (    s@   lib/python2.7/site-packages/dask/dataframe/tests/test_shuffle.pyt   test_compute_divisions¡  s    	!c      	   C   sß   t  j i t j j d ƒ d 6t j j d ƒ d 6t j j d ƒ d 6ƒ } t j | d d d d d t ƒ} t j j	 d	 t
 |  ƒ d
 d ƒ O | j d d d ƒ} | j ƒ  t d „  t j t
 |  ƒ ƒ Dƒ ƒ sÕ t ‚ Wd  QXd  S(   Nid   R   R   Rö   R   i
   Rm   RÈ   t   temporary_directoryR#   R   R   R   c         s   s   |  ] } | j  d  ƒ Vq d S(   s   .partdN(   t   endswith(   Rƒ   t   fn(    (    s@   lib/python2.7/site-packages/dask/dataframe/tests/test_shuffle.pys	   <genexpr>Á  s    (   R%   R   R&   R'   R   R(   Ri   R   RŸ   R   Rw   Rd   R)   R  t   ost   listdirR   (   t   tmpdirR1   R2   R{   (    (    s@   lib/python2.7/site-packages/dask/dataframe/tests/test_shuffle.pyt   test_temporary_directory·  s    !

c          C   sÜ   t  j i t t d ƒ ƒ d 6ƒ }  |  d d |  d <|  d j t ƒ |  d <t j |  d d ƒ} | j d ƒ } | j	 d d ƒ } | j
 d ƒ j ƒ  t | |  j d ƒ ƒ | j d ƒ } t | |  j d ƒ j d ƒ ƒ d  S(   Ni
   R   i   R   RI   R   i    (   R%   R   R+   R¶   R^   Rw   R   R(   Rd   RŒ   R  R)   R	   (   R1   R2   (    (    s@   lib/python2.7/site-packages/dask/dataframe/tests/test_shuffle.pyt   test_empty_partitionsÄ  s    "c          C   s'  d d f d d	 f d
 d f d d f d d f d d f d d f d d f g }  t t j f t t j f t t j f d „  t j d ƒ f g } x“ | D]‹ \ } } x| |  D]t \ } } g  | D]$ } | d  k rÒ | n	 | | ƒ ^ qº } g  | D] } | | ƒ ^ që } t | ƒ | k s§ t ‚ q§ Wq” Wd  S(   Ni   i   i   c         S   s   t  j |  d d ƒS(   NRÊ   RË   (   R%   RÎ   (   R   (    (    s@   lib/python2.7/site-packages/dask/dataframe/tests/test_shuffle.pyR   ä  R  t   NaT(   i   i   i   (   i   i   i   (   Ni   i   (   i   i   i   (   i   Ni   (   i   i   i   (   i   i   N(   i   i   i   (   i   i   NN(   i   i   i   i   (   NNi   i   (   i   i   i   i   (   i   NNi   (   i   i   i   i   (   Ni   Ni   Ni   N(   i   i   i   i   i   i   i   (	   R­   Rx   R&   t   nanRv   Rw   t
   datetime64R   R   (   t   testsR  R  t   none_valt   inputst   expectedR   t   params(    (    s@   lib/python2.7/site-packages/dask/dataframe/tests/test_shuffle.pyt   test_remove_nansÔ  s"    							1c       
   C   sû   t  j i t j d d ƒ d 6ƒ }  t  j i t j d d ƒ d 6ƒ } t j | d d ƒ} t j |  d d ƒ} t j j d	 d
 d d ƒ , | j	 | d d d d ƒ} | j
 ƒ  } Wd  QX|  j	 | d d d d ƒ} t j | j d ƒ j d t ƒ | ƒ d  S(   Ni    iPÃ  t   KEYi   iô  R   i   i  R   R   R#   R$   t   howt   innert   onRg   (   R%   R   R&   t   arangeR   R(   R   RŸ   R   t   mergeR)   t   tmt   assert_frame_equalt   sort_valuest   reset_indexRh   (   t   larget   smallt   dd_leftt   dd_rightt	   dd_mergedRŠ   R  (    (    s@   lib/python2.7/site-packages/dask/dataframe/tests/test_shuffle.pyt   test_gh_2730î  s    ""R¿   c            sç   t  j ƒ  ‰  ‡  f d †  ‰ d „  ‰ d } d } t | | d ƒ ‰ ‡ f d †  t | ƒ Dƒ } | j ‡ ‡ f d †  t | ƒ Dƒ ƒ t j | d ˆ d  d	 ƒ d  g | d	 ƒ } | j d d
 |  ƒt	 ˆ  ƒ } | | k sã t
 ‚ d  S(   Nc              s   t  ˆ  ƒ d  S(   N(   t   next(    (   t   count(    s@   lib/python2.7/site-packages/dask/dataframe/tests/test_shuffle.pyt	   increment  s    c         S   s3   t  j i t j j | ƒ d 6t j j | ƒ d 6ƒ S(   NR   R   (   R%   R   R&   R'   (   t   dummyRÁ   (    (    s@   lib/python2.7/site-packages/dask/dataframe/tests/test_shuffle.pyRÂ   
  s    g    €„.Ai2   i   c            s"   i  |  ] } ˆ  f d  | f “ q S(   t   inc(    (   Rƒ   R>   (   R4  (    s@   lib/python2.7/site-packages/dask/dataframe/tests/test_shuffle.pys
   <dictcomp>  s   	 c            s.   i  |  ]$ } ˆ  d  | f ˆ f d | f “ q S(   R6  R   (    (   Rƒ   R>   (   RÂ   RÁ   (    s@   lib/python2.7/site-packages/dask/dataframe/tests/test_shuffle.pys
   <dictcomp>  s   	 R   i   R   (   t	   itertoolsR3  Rx   R¶   t   updateR   R   R­   Rd   R2  R   (   R   RÄ   RÅ   RÙ   R2   t   ntimes(    (   R3  R4  RÂ   RÁ   s@   lib/python2.7/site-packages/dask/dataframe/tests/test_shuffle.pyt8   test_set_index_does_not_repeat_work_due_to_optimizations  s    	&,c          C   sŽ   t  j i d d d g d 6d d d g d 6d	 d
 d g d 6ƒ }  t j |  d d ƒ} | j d ƒ t j t ƒ  | j d d t ƒWd  QXd  S(   Ni	   i   i   R   i   i   i   R   i   i   i   RI   R   t   inplace(	   R%   R   R   R(   Rd   R    R¡   Rò   Rh   (   R1   R2   (    (    s@   lib/python2.7/site-packages/dask/dataframe/tests/test_shuffle.pyt(   test_set_index_errors_with_inplace_kwarg  s    	(   R   i    (   R   i   (   R   i   (   i   i   (i   R7  R  t   syst   pandasR%   R    R‘   t   numpyR&   R\   t   multiprocessingR³   R    t   pandas.util.testingt   utilt   testingR(  R   t   dask.dataframet	   dataframeR   R   t	   dask.baseR   R  R   R   R   R   R   R   t   dask.dataframe.utilsR	   R
   R   RÙ   t   IndexRØ   R   R)   R¦   R   t   markt   parametrizeR    R!   R4   R8   R=   R?   RC   RÍ   R­   Rã   R^   Rh   Ri   Rä   t   timedelta_rangeRW   R[   Rb   t   paramt   slowRe   Rf   Rj   Rz   R|   R   R‹   RŽ   R™   R£   R¥   R¬   R¯   t   skipift   version_infoR¾   R¸   RÀ   RÂ   RÆ   RÇ   RÉ   R×   RÚ   RÜ   Rá   Rì   Rñ   Rõ   Rú   Rû   Rÿ   R	  R  R  R  R  R!  R1  R:  R<  (    (    (    s@   lib/python2.7/site-packages/dask/dataframe/tests/test_shuffle.pyt   <module>   sª   ./,,,$$		$$
$$
,#&%%		?$!$
$$'					
				$		$$$		$				$										$