ó
¦–Õ\c           @` sY  d  d l  m Z m Z m Z d  d l Z d  d l m Z d  d l Z d  d l Z d  d l	 Z	 d  d l
 Z d  d l Z d  d l m Z d  d l m Z d d l m Z m Z m Z m Z m Z d d	 l m Z m Z d d
 l m Z m Z m Z d d l m Z d d l m  Z  d d l! m! Z! d d l" m# Z# m$ Z$ m% Z% d d e' e( d d d d „ Z) d „  Z* d e( d d d „ Z+ d d d d d „ Z, d d d „ Z- d d d d d „ Z. d e/ f d „  ƒ  YZ0 d e' d „ Z1 d d d „ Z2 d „  Z3 d „  Z4 d „  Z5 d „  Z6 d  „  Z7 d! „  Z8 d" „  Z9 d# „  Z: d$ „  Z; d% „  Z< e( d d& „ Z= d' „  Z> d S((   i    (   t   absolute_importt   divisiont   print_functionN(   t   getitem(   t   groupsort_indexer(   t   hash_pandas_objecti   (   t	   DataFramet   Seriest   _Framet   _concatt   map_partitionsi   (   t   baset   config(   t   tokenizet   computet   compute_as_if_collection(   t   delayed(   t   HighLevelGraph(   t   sizeof(   t   digitt   insertt   Mg      ð?g    €„žAc	         K` sÕ  t  | t ƒ r( | j |  j j k r( |  St  | t t t f ƒ rY t d t | ƒ ƒ ‚ n  | d k r€ t	 }
 t
 d |  j ƒ } n | d k r˜ |  j } n  t }
 t  | t ƒ sº |  | } n | } | d k r°|
 r$t j | |  ƒ \ } }  |  j d t ƒ } g  | D] } t t ƒ | ƒ ^ q} n t j | ƒ \ } g  } | j | d | ƒ} | j d t ƒ } g  | D] } | j ƒ  ^ qj} g  | D] } | j
 ƒ  ^ q‰} t j | | | ƒ \ } } } t j | | | | d t ƒ\ } } } } | j ƒ  } t j | ƒ j ƒ  } |
 s| rt | ƒ } t
 t j | | ƒ d ƒ } t | |  j ƒ } t | ƒ } yR t j d t j  d | d | d ƒ d	 t j  d | d | ƒ d
 | ƒ j ƒ  } Wqt! t" f k
 r
t j  d | d | d ƒ j# t$ ƒ } g  | D] } | | ^ qñ} qXn  t% | ƒ } t% | ƒ } | t& | ƒ k r°| t& | ƒ k r°t d „  t' | d  | d ƒ Dƒ ƒ r°| | d g } t( |  | d | d | ƒ} | j) t* j+ ƒ Sn  t, |  | | d | d | d | |	 S(   s$    See _Frame.set_index for docstring s~   Dask dataframe does not yet support multi-indexes.
You tried to index with this index: %s
Indexes must be single columns only.t   autoid   t   optimize_grapht   upsamplei   t   xi    t   xpt   fpc         s` s!   |  ] \ } } | | k  Vq d  S(   N(    (   t   .0t   mxt   mn(    (    s5   lib/python2.7/site-packages/dask/dataframe/shuffle.pys	   <genexpr>U   s    iÿÿÿÿt   dropt	   divisionst   shuffleR   N(-   t
   isinstanceR   t   _namet   indexR   t   tuplet   listt   NotImplementedErrort   strt   Truet   maxt   npartitionst   Nonet   FalseR   t   optimizet
   to_delayedR   R   t   _repartition_quantilest   minR   t   tolistt   pdt   isnullt   allt   sumt   matht   ceilt   lent   npt   interpt   linspacet	   TypeErrort
   ValueErrort   astypet   intt   remove_nanst   sortedt   zipt   set_sorted_indexR
   R   t
   sort_indext   set_partition(   t   dfR$   R+   R!   R   R   R   R    t   partition_sizet   kwargst   repartitiont   index2t   partst   partt   sizest   ipartst   ipartt   minst   maxest   empty_dataframe_detectedt   totalt   nt   indexest   it   result(    (    s5   lib/python2.7/site-packages/dask/dataframe/shuffle.pyt	   set_index   sh    $(&&$$'c         C` sÏ   t  |  ƒ }  xL t t |  ƒ d d d ƒ D]. } t j |  | ƒ r) |  | d |  | <q) q) Wxm t t |  ƒ d d d ƒ D]O } t j |  | ƒ sx x/ t | d t |  ƒ ƒ D] } |  | |  | <q« WPqx qx W|  S(   s   Remove nans from divisions

    These sometime pop up when we call min/max on an empty partition

    Examples
    --------
    >>> remove_nans((np.nan, 1, 2))
    [1, 1, 2]
    >>> remove_nans((1, np.nan, 2))
    [1, 2, 2]
    >>> remove_nans((1, 2, np.nan))
    [1, 2, 2]
    i   iÿÿÿÿi   (   R&   t   rangeR9   R3   R4   (   R    RW   t   j(    (    s5   lib/python2.7/site-packages/dask/dataframe/shuffle.pyRA   `   s    ## i    c         C` s;  t  j | ƒ rO |  | j t d | d t j d g ƒ ƒ} |  j d | ƒ } n? | j t d | d t j d g ƒ ƒ} |  j d | d | ƒ } t | d d | d t | ƒ d d	 | d
 | ƒ}	 t  j | ƒ rø |	 j t	 d | d | d |  j
 j ƒ}
 n* |	 j t d | j d | d |  j
 j ƒ}
 | |
 _ |
 j t j ƒ S(   s(   Group DataFrame by index

    Sets a new index and partitions data along that index according to
    divisions.  Divisions are often found by computing approximate quantiles.
    The function ``set_index`` will do both of these steps.

    Parameters
    ----------
    df: DataFrame/Series
        Data that we want to re-partition
    index: string or Series
        Column to become the new index
    divisions: list
        Values to form new divisions between partitions
    drop: bool, default True
        Whether to delete columns to be used as the new index
    shuffle: str (optional)
        Either 'disk' for an on-disk shuffle or 'tasks' to use the task
        scheduling framework.  Use 'disk' if you are on a single machine
        and 'tasks' if you are on a distributed cluster.
    max_branch: int (optional)
        If using the task-based shuffle, the amount of splitting each
        partition undergoes.  Increase this for fewer copies but more
        scheduler overhead.

    See Also
    --------
    set_index
    shuffle
    partd
    R    t   metai    t   _partitionst   _indext
   max_branchR+   i   R!   R   t
   index_nameR   t   column_dtype(   R:   t   isscalarR
   t   set_partitions_preR3   R   t   assignt   rearrange_by_columnR9   t   set_index_post_scalart   columnst   dtypet   set_index_post_seriest   nameR    R   RE   (   RG   R$   R    R_   R   R!   R   t
   partitionst   df2t   df3t   df4(    (    s5   lib/python2.7/site-packages/dask/dataframe/shuffle.pyRF   }   s&    !		c   	      C` s³   t  | t ƒ s! |  j | ƒ } n  | j t d | p9 |  j d t j d g ƒ d t ƒ} |  j	 d | ƒ } |  j
 j j | j
 j _ t | d d | d | d | d | ƒ} | d =| S(	   sç   Group DataFrame by index

    Hash grouping of elements. After this operation all elements that have
    the same index will be in the same partition. Note that this requires
    full dataset read, serialization and shuffle. This is expensive. If
    possible you should avoid shuffles.

    This does not preserve a meaningful index/partitioning scheme. This is not
    deterministic if done in parallel.

    See Also
    --------
    set_index
    set_partition
    shuffle_disk
    R+   R\   i    t   transform_divisionsR]   R_   R!   R   (   R"   R   t   _select_columns_or_indexR
   t   partitioning_indexR+   R3   R   R-   Rd   t   _metaR$   Rj   Re   (	   RG   R$   R!   R+   R_   R   Rk   Rl   Rm   (    (    s5   lib/python2.7/site-packages/dask/dataframe/shuffle.pyR!   ¹   s    		c      	   C` ss   |  | j  t d | d t j d g ƒ ƒ} |  j d | ƒ } t | d d | d t | ƒ d d | ƒ} | d =| S(	   s<    Shuffle dataframe so that column separates along divisions R    R\   i    R]   R_   R+   i   R!   (   R
   Rc   R3   R   Rd   Re   R9   (   RG   t   columnR    R_   R!   Rk   Rl   Rm   (    (    s5   lib/python2.7/site-packages/dask/dataframe/shuffle.pyt   rearrange_by_divisionsÛ   s    c         C` ss   | p t  j d d  ƒ p d } | d k r@ t |  | | d | ƒS| d k r_ t |  | | | ƒ St d | ƒ ‚ d  S(   NR!   t   diskR   t   taskss   Unknown shuffle method %s(   R   t   getR,   t   rearrange_by_column_diskt   rearrange_by_column_tasksR'   (   RG   t   colR+   R_   R!   R   (    (    s5   lib/python2.7/site-packages/dask/dataframe/shuffle.pyRe   ê   s    t   maybe_buffered_partdc           B` s/   e  Z d  Z e d d „ Z d „  Z d „  Z RS(   s[   
    If serialized, will return non-buffered partd. Otherwise returns a buffered partd
    c         C` s(   | p t  j d d  ƒ |  _ | |  _ d  S(   Nt   temporary_directory(   R   Rw   R,   t   tempdirt   buffer(   t   selfR~   R}   (    (    s5   lib/python2.7/site-packages/dask/dataframe/shuffle.pyt   __init__ù   s    c         C` s-   |  j  r t t |  j  f f St t f f Sd  S(   N(   R}   R{   R-   (   R   (    (    s5   lib/python2.7/site-packages/dask/dataframe/shuffle.pyt
   __reduce__ý   s    	c         O` sr   d d  l  } |  j r- | j d |  j ƒ } n | j ƒ  } |  j ra | j | j | j ƒ  | ƒ ƒ S| j | ƒ Sd  S(   Ni    t   dir(   t   partdR}   t   FileR~   t   PandasBlockst   Buffert   Dict(   R   t   argsRI   Rƒ   t   file(    (    s5   lib/python2.7/site-packages/dask/dataframe/shuffle.pyt   __call__  s    		N(   t   __name__t
   __module__t   __doc__R)   R,   R€   R   RŠ   (    (    (    s5   lib/python2.7/site-packages/dask/dataframe/shuffle.pyR{   õ   s   	c         ` sÀ  ˆ d	 k r ˆ j ‰ n  t ˆ ˆ ˆ ƒ } t j ƒ  j } d | f ‰ i t ƒ  f ˆ 6} d | ‰ ‡ ‡ ‡ ‡ f d †  t ˆ j ƒ  ƒ Dƒ } g  } i  }	 | rt	 j
 ˆ j | | ƒ }
 ˆ t | ƒ g } t t |
 | ƒ \ } } i | ˆ 6} t t t | ƒ | ƒ ƒ } n | j ˆ ƒ d | ‰  i t t | ƒ f ˆ  6} d | ‰ ‡  ‡ ‡ ‡ f d †  t ˆ ƒ Dƒ } d
 ˆ d } t j
 | | | | ƒ }	 t	 j ˆ |	 d | ƒ}
 t |
 ˆ ˆ j | ƒ S(   sº    Shuffle using local disk

    See Also
    --------
    rearrange_by_column_tasks:
        Same function, but using tasks rather than partd
        Has a more informative docstring
    s   zpartd-s   shuffle-partition-c         ` s4   i  |  ]* \ } } t  | ˆ  ˆ ˆ f ˆ | f “ q S(    (   t   shuffle_group_3(   R   RW   t   key(   Rs   Rj   R+   t   p(    s5   lib/python2.7/site-packages/dask/dataframe/shuffle.pys
   <dictcomp>#  s   	s   barrier-s   shuffle-collect-c         ` s1   i  |  ]' } t  ˆ | ˆ j ˆ  f ˆ | f “ q S(    (   t   collectRr   (   R   RW   (   t   barrier_tokenRG   Rj   R   (    s5   lib/python2.7/site-packages/dask/dataframe/shuffle.pys
   <dictcomp>7  s   	i   t   dependenciesN(   N(   R,   R+   R   t   uuidt   uuid1t   hexR{   t	   enumeratet   __dask_keys__R   t   merget   daskRB   R   R   t   dictRC   t   appendt   barrierR&   RZ   t   toolzt   from_collectionsRr   (   RG   Rs   R+   R   t   tokent   always_new_tokent   dsk1t   dsk2R“   t   layert   grapht   keyst   ppt   valuest   dsk3t   dsk4R    (    (   R’   Rs   RG   Rj   R+   R   s5   lib/python2.7/site-packages/dask/dataframe/shuffle.pyRx     s6    	


c         ` s.  | p	 d } ˆ j  ‰ t t j t j ˆ ƒ t j | ƒ ƒ ƒ ‰ ˆ d k rl t t j ˆ d ˆ ƒ ƒ ‰ n ˆ ‰ g  } g  } g  } g  t ˆ ˆ ƒ D]+ ‰ t ‡ ‡ f d †  t ˆ ƒ Dƒ ƒ ^ q• ‰ t ˆ ˆ  | ƒ ‰ ‡ ‡ f d †  t ˆ ƒ Dƒ } x¢ t d ˆ d ƒ D] ‰ ‡  ‡ ‡ ‡ ‡ f d †  ˆ Dƒ } ‡ ‡ ‡ f d †  t ˆ ƒ Dƒ }	 ‡ ‡ ‡ f d †  ˆ Dƒ }
 | j	 | ƒ | j	 |	 ƒ | j	 |
 ƒ qW‡ ‡ f d †  t ˆ ƒ Dƒ } t
 j | | | | | Œ } t j d	 ˆ | d
 ˆ g ƒ} t | d	 ˆ ˆ ˆ j ƒ } | d k	 r| ˆ j  k rg  t | ƒ D] ‰ ˆ ˆ j  ^ q;} t | | ƒ ‰ ‡  ‡ f d †  t | j ƒ  ƒ Dƒ } x? t | ƒ D]1 } t d ˆ | | f | f | d ˆ | f <q•Wt j d ˆ | d
 | g ƒ} t | d ˆ | d g | d ƒ } n | } d ˆ j  d | _ | S(   s‹   Order divisions of DataFrame so that all values within column align

    This enacts a task-based shuffle.  It contains most of the tricky logic
    around the complex network of tasks.  Typically before this function is
    called a new column, ``"_partitions"`` has been added to the dataframe,
    containing the output partition number of every row.  This function
    produces a new dataframe where every row is in the proper partition.  It
    accomplishes this by splitting each input partition into several pieces,
    and then concatenating pieces from different input partitions into output
    partitions.  If there are enough partitions then it does this work in
    stages to avoid scheduling overhead.

    Parameters
    ----------
    df: dask.dataframe.DataFrame
    column: str
        A column name on which we want to split, commonly ``"_partitions"``
        which is assigned by functions upstream
    max_branch: int
        The maximum number of splits per input partition.  Defaults to 32.
        If there are more partitions than this then the shuffling will occur in
        stages in order to avoid creating npartitions**2 tasks
        Increasing this number increases scheduling overhead but decreases the
        number of full-dataset transfers that we have to make.
    npartitions: Optional[int]
        The desired number of output partitions

    Returns
    -------
    df3: dask.dataframe.DataFrame

    See also
    --------
    rearrange_by_column_disk: same operation, but uses partd
    rearrange_by_column: parent function that calls this or rearrange_by_column_disk
    shuffle_group: does the actual splitting per-partition
    i    i   c         3` s!   |  ] } t  ˆ  | ˆ ƒ Vq d  S(   N(   R   (   R   R[   (   RW   t   k(    s5   lib/python2.7/site-packages/dask/dataframe/shuffle.pys	   <genexpr>u  s    c         ` sM   i  |  ]C \ } } | ˆ  j  k  r0 ˆ  j | f n ˆ  j d  ˆ d | f “ q S(   s   shuffle-join-i    (   R+   R#   Rr   (   R   RW   t   inp(   RG   R    (    s5   lib/python2.7/site-packages/dask/dataframe/shuffle.pys
   <dictcomp>z  s   	c         ` sM   i  |  ]C } t  d  ˆ ˆ d | f ˆ  ˆ d ˆ ˆ f d ˆ ˆ | f “ q S(   s   shuffle-join-i   s   shuffle-group-(   t   shuffle_group(   R   R¬   (   Rs   R«   RU   t   stageR    (    s5   lib/python2.7/site-packages/dask/dataframe/shuffle.pys
   <dictcomp>  s   	c         ` sL   i  |  ]B } ˆ  D]5 } t  d  ˆ ˆ | f | f d ˆ ˆ | | f “ q q S(   s   shuffle-group-s   shuffle-split-(   R   (   R   RW   R¬   (   t   inputsR®   R    (    s5   lib/python2.7/site-packages/dask/dataframe/shuffle.pys
   <dictcomp>…  s   	c         ` sm   i  |  ]c } t  g  t ˆ  ƒ D]4 } d  ˆ ˆ | ˆ d t | ˆ d | ƒ f ^ q f d ˆ ˆ | f “ q S(   s   shuffle-split-i   s   shuffle-join-(   R	   RZ   R   (   R   R¬   R[   (   R«   R®   R    (    s5   lib/python2.7/site-packages/dask/dataframe/shuffle.pys
   <dictcomp>Œ  s   	c         ` s6   i  |  ], \ } } d  ˆ ˆ  | f d ˆ | f “ q S(   s   shuffle-join-s   shuffle-(    (   R   RW   R¬   (   t   stagesR    (    s5   lib/python2.7/site-packages/dask/dataframe/shuffle.pys
   <dictcomp>˜  s   	s   shuffle-R“   c         ` s2   i  |  ]( \ } } t  | ˆ  f d  ˆ | f “ q S(   s   repartition-group-(   t   shuffle_group_2(   R   RW   R«   (   Rs   R    (    s5   lib/python2.7/site-packages/dask/dataframe/shuffle.pys
   <dictcomp>¥  s   	s   repartition-group-s   repartition-get-N(   N(   R+   R@   R7   R8   t   logRZ   R%   R   R—   Rœ   Rž   R™   R   RŸ   R   R    R,   R˜   t   shuffle_group_get(   RG   Rs   R_   R+   t   groupst   splitst   joinst   startt   groupt   splitt   joint   endt   dskR¥   Rl   RL   R   t   graph2Rm   (    (	   Rs   RG   RW   R¯   R«   RU   R®   R°   R    s5   lib/python2.7/site-packages/dask/dataframe/shuffle.pyRy   B  sR    &	+ ?

&/'c         C` s   t  |  d t ƒt | ƒ S(   s~  
    Computes a deterministic index mapping each record to a partition.

    Identical rows are mapped to the same partition.

    Parameters
    ----------
    df : DataFrame/Series/Index
    npartitions : int
        The number of partitions to group into.

    Returns
    -------
    partitions : ndarray
        An array of int64 values mapping each record to a partition.
    R$   (   R   R-   R@   (   RG   R+   (    (    s5   lib/python2.7/site-packages/dask/dataframe/shuffle.pyRq   ¹  s    c         C` s   t  |  ƒ d S(   Ni    (   R&   (   Rˆ   (    (    s5   lib/python2.7/site-packages/dask/dataframe/shuffle.pyR   Í  s    
c         C` s)   |  j  | ƒ } t | ƒ d k r% | S| S(   s1    Collect partitions from partd, yield dataframes i    (   Rw   R9   (   R   RM   R\   R’   t   res(    (    s5   lib/python2.7/site-packages/dask/dataframe/shuffle.pyR‘   Ò  s    c         C` sG   t  j | ƒ j |  d d ƒd } t | ƒ d | |  | d k j <| S(   Nt   sidet   righti   i   iÿÿÿÿ(   R3   R   t   searchsortedR9   R¨   (   t   sR    Rk   (    (    s5   lib/python2.7/site-packages/dask/dataframe/shuffle.pyRc   Ø  s    "!c         C` sá   t  |  ƒ s i  |  f S|  | j j t j ƒ } | j ƒ  d } t | j t j ƒ | ƒ \ } } |  j | ƒ } | j	 ƒ  } g  t
 | d  | d ƒ D] \ } } | j | | !^ q“ }	 t t
 t | ƒ |	 ƒ ƒ }
 |
 |  j d  f S(   Ni   iÿÿÿÿi    (   R9   t   _valuesR?   R:   t   int64R*   R   t   viewt   taket   cumsumRC   t   ilocR›   RZ   (   RG   Rz   t   indRU   t   indexert	   locationsRl   t   at   bRL   t   result2(    (    s5   lib/python2.7/site-packages/dask/dataframe/shuffle.pyR±   Þ  s    
!:c         C` s(   |  \ } } | | k r  | | S| Sd  S(   N(    (   t   g_headRW   t   gt   head(    (    s5   lib/python2.7/site-packages/dask/dataframe/shuffle.pyR³   ë  s    c         C` s+  | d k r |  | } n t  |  | d t ƒ} | j } t j | d ƒ } t j | | ƒ j | d t ƒ} t j | | | d | ƒt j | | d | ƒt | j t j	 ƒ | ƒ \ } }	 |  j
 | ƒ }
 |	 j ƒ  }	 g  t |	 d  |	 d ƒ D] \ } } |
 j | | !^ qð } t t t | ƒ | ƒ ƒ S(   s2   Splits dataframe into groups

    The group is determined by their final partition, and which stage we are in
    in the shuffle

    Parameters
    ----------
    df: DataFrame
    col: str
        Column name on which to split the dataframe
    stage: int
        We shuffle dataframes with many partitions we in a few stages to avoid
        a quadratic number of tasks.  This number corresponds to which stage
        we're in, starting from zero up to some small integer
    k: int
        Desired number of splits from this dataframe
    npartition: int
        Total number of output partitions for the full dataframe

    Returns
    -------
    out: Dict[int, DataFrame]
        A dictionary mapping integers in {0..k} to dataframes such that the
        hash values of ``df[col]`` are well partitioned.
    R]   R$   i   t   copyt   outiÿÿÿÿi   (   R   R-   RÃ   R:   t   min_scalar_typet   modR?   t   floor_divideR   RÄ   RÆ   RÇ   RC   RÈ   R›   RZ   (   RG   Rz   R®   R«   R+   RÉ   t   ct   typRÊ   RË   Rl   RÌ   RÍ   RL   (    (    s5   lib/python2.7/site-packages/dask/dataframe/shuffle.pyR­   ó  s    	!!:c         ` s?   |  j  | ƒ ‰  ‡  f d †  ˆ  j Dƒ } | j | d t ƒd  S(   Nc         ` s"   i  |  ] } ˆ  j  | ƒ | “ q S(    (   t	   get_group(   R   RW   (   RÐ   (    s5   lib/python2.7/site-packages/dask/dataframe/shuffle.pys
   <dictcomp>#  s   	 t   fsync(   t   groupbyR´   Rœ   R)   (   RG   Rz   R+   R   t   d(    (   RÐ   s5   lib/python2.7/site-packages/dask/dataframe/shuffle.pyRŽ   !  s    c         C` s=   |  j  d d d ƒj | d | ƒ} | j j | ƒ | _ | S(   NR]   t   axisi   R   (   R   RY   Rg   R?   (   RG   R`   R   Ra   Rl   (    (    s5   lib/python2.7/site-packages/dask/dataframe/shuffle.pyRf   '  s    $c         C` sI   |  j  d d d ƒj d d t ƒ} | | j _ | j j | ƒ | _ | S(   NR]   RÝ   i   R^   R   (   R   RY   R)   R$   Rj   Rg   R?   (   RG   R`   R   Ra   Rl   (    (    s5   lib/python2.7/site-packages/dask/dataframe/shuffle.pyRi   -  s    $c         K` sÁ   t  | t ƒ s* |  j j | d | ƒ} n |  j j | j d | ƒ} t t j |  | d | d | ƒ} | s~ t | |  } n0 t | ƒ t |  j ƒ k r® d } t	 | ƒ ‚ n  t
 | ƒ | _ | S(   NR   R\   se  When doing `df.set_index(col, sorted=True, divisions=...)`, divisions indicates known splits in the index column. In this case divisions must be the same length as the existing divisions in `df`

If the intent is to repartition into new divisions after setting the index, you probably want:

`df.set_index(col, sorted=True).repartition(divisions=divisions)`(   R"   R   Rr   RY   R
   R   t   compute_divisionsR9   R    R>   R%   (   RG   R$   R   R    RI   R\   RX   t   msg(    (    s5   lib/python2.7/site-packages/dask/dataframe/shuffle.pyRD   4  s    !c         K` s  |  j  j t j d |  j  ƒ} |  j  j t j d |  j  ƒ} t | | |  \ } } t | ƒ t | ƒ k s£ t | ƒ t | ƒ k s£ t d „  t	 | | ƒ Dƒ ƒ r¸ t
 d | | ƒ ‚ n  t d „  t	 | d  | d ƒ Dƒ ƒ rï t j d ƒ n  t | ƒ t | ƒ d f } | S(   NR\   c         s` s!   |  ] \ } } | | k Vq d  S(   N(    (   R   RÌ   RÍ   (    (    s5   lib/python2.7/site-packages/dask/dataframe/shuffle.pys	   <genexpr>S  s    s2   Partitions must be sorted ascending with the indexc         s` s!   |  ] \ } } | | k Vq d  S(   N(    (   R   RÌ   RÍ   (    (    s5   lib/python2.7/site-packages/dask/dataframe/shuffle.pys	   <genexpr>W  s    i   s   Partition indices have overlap.iÿÿÿÿ(   R$   R
   R   R1   R*   R   RB   R&   t   anyRC   R>   t   warningst   warnR%   (   RG   RI   RQ   RR   R    (    (    s5   lib/python2.7/site-packages/dask/dataframe/shuffle.pyRÞ   L  s    '(?   t
   __future__R    R   R   R7   t   operatorR   R”   Rá   Rž   t   numpyR:   t   pandasR3   t   pandas._libs.algosR   t   pandas.utilR   t   coreR   R   R   R	   R
   t    R   R   R   R   R   R   t   highlevelgraphR   R   t   utilsR   R   R   R,   R-   R)   RY   RA   RF   R!   Rt   Re   t   objectR{   Rx   Ry   Rq   R   R‘   Rc   R±   R³   R­   RŽ   Rf   Ri   RD   RÞ   (    (    (    s5   lib/python2.7/site-packages/dask/dataframe/shuffle.pyt   <module>   sT   (		F		;	!
3w							.			