ó
¦–Õ\c        	   @   sq  d  d l  Z  d  d l Z d  d l Z d  d l j j Z d  d l	 Z	 d  d l
 Z
 d  d l j Z d  d l m Z d  d l m Z m Z m Z m Z g  Z x` e e g D]R Z e j e j e d ƒ d e ƒƒ Z e j e d d ƒZ e j e e f ƒ q¢ We j e  d ƒ d	 e j e d ƒ ƒ ƒZ e j e d d ƒZ e j e j! ƒ  j" e j" f ƒ e j# i e d
 ƒ d 6e d ƒ d 6e j$ d ƒ d 6e d ƒ d 6e j$ d d d ƒd 6ƒ Z% e j# i e d ƒ d 6e d ƒ d 6e j$ d d ƒ d 6e d ƒ d 6e j$ d d d d ƒd 6ƒ Z& e j# i e d ƒ d 6e d ƒ d 6e j$ d d ƒ d 6e d ƒ d 6e j$ d d d d ƒd 6ƒ Z' e% e& e' g Z( g  Z) x e( D]y Z* e* j+ j, d ƒ e* _+ e* j- j, d ƒ e* _- e) j e* j. d e* j+ j/ j0 e d ƒ ƒ d e* j- j/ j0 e d ƒ ƒ ƒ ƒ q’Wg  e( D] Z1 e1 j2 e1 j- ƒ ^ qZ3 g  e) D] Z1 e1 j2 e1 j- ƒ ^ q;Z4 g  e( D]! Z1 e1 j2 e1 j- e1 j5 g ƒ ^ q`Z6 g  e) D]! Z1 e1 j2 e1 j- e1 j5 g ƒ ^ qŽZ7 d  „  Z8 d! „  Z9 d" „  Z: d# „  Z; d$ „  Z< e	 j= j> d% d& d' g ƒ d( „  ƒ Z? e	 j= j> d d) d* g ƒ d+ „  ƒ Z@ d, „  ZA d- „  ZB d. „  ZC e d/ „ ZD d0 „  ZE d1 d3 d2 „  ƒ  YZF d S(4   iÿÿÿÿN(   t   _concat(   t	   make_metat	   assert_eqt   is_categorical_dtypet   clear_known_categoriest   bacbact   orderedt   npartitionsi   i   t   indext   abcdet   vt   xxxxxt   wi   t   xt   abcbct   yt   dtypet   f8t   zt   fghijt   yyyyyi
   t   abbbat   klmnot   zzzzzi   t   bcbcct   categoryt   xyzt   abcc          C   s1  t  j t t ƒ t j t ƒ ƒ t  j t g  t D] }  |  j ^ q/ ƒ t j g  t D] }  |  j ^ qQ ƒ ƒ t  j	 t g  t
 D] }  |  j ^ qz ƒ t j g  t D] }  |  ^ qœ ƒ j ƒ t  j t t
 ƒ t j t ƒ ƒ t  j t g  t
 D] }  |  d d g ^ qä ƒ t j g  t D] }  |  d d g ^ qƒ ƒ t  j t g  t
 D] }  |  j ^ q=ƒ t j g  t D] }  |  j ^ q_ƒ ƒ t  j t g  t
 D] }  |  j ^ qˆƒ t j g  t D] }  |  j ^ qªƒ ƒ t  j	 t g  t D] }  |  j ^ qÓƒ t j g  t D] }  |  ^ qõƒ j ƒ t  j t t ƒ t j t ƒ ƒ d  S(   NR   R   (   t   tmt   assert_frame_equalR    t   framest   pdt   concatt   frames2t   assert_series_equalR   t   assert_index_equalt   frames3R   t   frames4R   R   t   frames5t   frames6(   t   i(    (    sD   lib/python2.7/site-packages/dask/dataframe/tests/test_categorical.pyt   test_concat_unions_categoricals8   s    %&%&,-%&%&%&c          C   sE  t  j d „  t t ƒ Dƒ d t i d d 6d d 6d d 6d d	 6d
 d 6ƒ d  g d ƒ }  |  j ƒ  } t |  j j	 ƒ  | j j	 ƒ  ƒ t |  j j
 ƒ  | j j
 ƒ  ƒ t |  j |  j ƒ j ƒ  | j | j ƒ j ƒ  ƒ t |  j |  j ƒ j j
 ƒ  | j | j ƒ j j
 ƒ  ƒ t |  j j |  j ƒ j ƒ  | j j | j ƒ j ƒ  ƒ d  S(   Nc         S   s%   i  |  ] \ } } | d  | f “ q S(   t   unknown(    (   t   .0R(   t   df(    (    sD   lib/python2.7/site-packages/dask/dataframe/tests/test_categorical.pys
   <dictcomp>\   s   	 R*   t   objectR
   R   R   t   i8R   R   R   R   i   (   t   ddt	   DataFramet	   enumerateR   R   t   Nonet   computeR   R   t   value_countst   nuniquet   groupbyt   sumR   t   count(   t   ddfR,   (    (    sD   lib/python2.7/site-packages/dask/dataframe/tests/test_categorical.pyt   test_unknown_categoricals[   s    17c          C   s®   t  j i t  j d d d d g ƒ d 6d d d d g d 6ƒ }  t |  d ƒ sT t ‚ t |  d ƒ sk t ‚ t j |  d ƒ } t | d ƒ s“ t ‚ t | d ƒ sª t ‚ d  S(   Ni   i   i   i   t   catR   (   R   R0   t   CategoricalR   t   AssertionErrorR/   t   from_pandas(   R,   R9   (    (    sD   lib/python2.7/site-packages/dask/dataframe/tests/test_categorical.pyt   test_is_categorical_dtypel   s    %c          C   sì  t  t d ƒ j d i d d 6ƒ }  t j d „  t t ƒ Dƒ d |  d  g d ƒ j d i d d 6ƒ } | j d | j	 j
 j d	 d d
 g ƒ ƒ } | j	 j
 j s¤ t ‚ | j j
 j sº t ‚ | j j
 j sÐ t ‚ | j ƒ  } xd  t t g D]ô} | t k	 } | j d | ƒ } | j j
 j s%t ‚ | j j
 j s:t ‚ | j j
 j | k sUt ‚ t | | j i d d 6ƒ d t ƒ| j d | d d ƒ } | j j
 j s¥t ‚ | j j
 j sºt ‚ | j j
 j | k sÕt ‚ t | | j i d d 6ƒ d t ƒ| j d d | ƒ} | j j
 j s#t ‚ | j j
 j s8t ‚ | j j
 j | k sSt ‚ t | | j i d d 6ƒ d t ƒ| j d d | ƒ} | j j
 j s t ‚ | j j d k s¸t ‚ | j j
 j | k sÓt ‚ t | | ƒ qì W| j d g  d t ƒ } | j j
 j st ‚ t | | ƒ | j d g d t ƒ| k sBt ‚ | j g  d t ƒ| k sct ‚ | j d g ƒ | k st ‚ | j g  ƒ | k sœt ‚ t j t ƒ  | j d d ƒ Wd  QXt j t ƒ  | j d d ƒ Wd  QXd  S(   Ni    t   columnst   y_R   c         S   s%   i  |  ] \ } } | d  | f “ q S(   R*   (    (   R+   R(   R,   (    (    sD   lib/python2.7/site-packages/dask/dataframe/tests/test_categorical.pys
   <dictcomp>}   s   	 R*   i   R   R   R   R   R   R
   t   check_categoricalt   split_everyi   R-   i   t   foo(   R   R%   t   renameR/   R0   R1   R$   R2   t   assignR   R;   t   set_categoriest   knownR=   RA   R   R3   t   Truet   Falset
   categorizeR
   R   t   astypeR   t   pytestt   raisest
   ValueError(   t   metaR9   R,   R   t   known_indext   ddf2t   ddf_known_index(    (    sD   lib/python2.7/site-packages/dask/dataframe/tests/test_categorical.pyt   test_categorizey   sR    #)*###$!c          C   s;  t  j t j ƒ  d d ƒ}  |  j ƒ  } |  j ƒ  } | j j j sH t	 ‚ t
 | | j t j | j ƒ ƒ d t d t ƒ|  j d t ƒ |  k s” t	 ‚ t  j | j | j j d ƒ ƒ d d ƒ}  |  j ƒ  } |  j d t ƒ } | j j j sñ t	 ‚ t
 | | j t j | j ƒ ƒ d t d t ƒ|  j ƒ  |  k s7t	 ‚ d  S(   NR   i   t   check_divisionsRB   R   t   idx(   R/   R>   R   t   makeDataFrameR3   RK   R   R;   RH   R=   R   t	   set_indexR   t   CategoricalIndexRJ   t   ARE   RI   (   R9   R,   RR   (    (    sD   lib/python2.7/site-packages/dask/dataframe/tests/test_categorical.pyt   test_categorize_index´   s    !*!t   shufflet   diskt   tasksc      
   C   sO  t  j i d d d d g d 6d d d d g d	 6ƒ } t  j | d	 d
 d d d g d t ƒ| d	 <t j | d d ƒ} t j j d d d |  ƒ ¸| j	 d	 d | j
 ƒ} | j d ƒ | j d ƒ } } t | j j ƒ  ƒ d g k sí t ‚ t t | j j ƒ  ƒ ƒ d d d g k st ‚ | j	 | j d | j
 ƒ} | j d ƒ | j d ƒ } } t | j j ƒ  ƒ d g k s{t ‚ t t | j j ƒ  ƒ ƒ d d d g k s«t ‚ | j	 d	 d d d d g d | j
 ƒ} | j d ƒ | j d ƒ } } t | j j ƒ  ƒ d g k st ‚ t t | j j ƒ  ƒ ƒ d d d g k sEt ‚ Wd  QXd  S(   Ni   i   i   i   R   t   at   bt   cR   t
   categoriesR   R   t	   schedulert   syncR\   i    t	   divisions(   R   R0   R<   RI   R/   R>   t   daskt   configt   setRX   R   t   get_partitiont   listR   R3   R=   t   sortedR   (   R\   R,   R_   R`   t   d1t   d2(    (    sD   lib/python2.7/site-packages/dask/dataframe/tests/test_categorical.pyt   test_categorical_set_indexÌ   s"    5,$0$0$i   i   c         C   s¨   t  j i t d ƒ d 6t d ƒ d 6ƒ } t j | d d ƒ} | d j d ƒ | d <| j d |  ƒ } | j ƒ  } | d j d ƒ | d <t	 | | ƒ t	 | | ƒ d  S(   Ni
   R   t
   abababcbcbR   R   i   R   (
   R   R0   t   rangeRj   R/   R>   RL   t   repartitiont   copyR   (   R   R,   R9   RR   (    (    sD   lib/python2.7/site-packages/dask/dataframe/tests/test_categorical.pyt    test_repartition_on_categoricalsä   s    )c          C   s$  t  j i t d d d ƒ d 6t d ƒ d 6ƒ }  |  j j d ƒ |  _ t j |  d	 d
 ƒ} d t | j ƒ k sv t	 ‚ d t | j
 ƒ k s‘ t	 ‚ t | j d ƒ s© t	 ‚ t | j
 d ƒ sÂ t	 ‚ |  j |  j ƒ } t j | d	 d
 d t ƒ} t | j d ƒ st	 ‚ t | j d ƒ s t	 ‚ d  S(   NR_   i   R`   Ra   R   i   R   R   R   i   R;   t   sortRb   t   aaaaat   bbbbbt   ccccc(   R   R0   Rj   Rp   R   RL   R/   R>   t   dirR=   R   t   hasattrRX   RJ   R   (   R,   R9   t   df2RR   (    (    sD   lib/python2.7/site-packages/dask/dataframe/tests/test_categorical.pyt"   test_categorical_accessor_presenceñ   s    1c          C   s{   t  j t j i d d d t d ƒ g d 6ƒ d d ƒ}  t j d  ƒ  } |  j ƒ  j	 ƒ  Wd  QXt
 | ƒ d k sw t ‚ d  S(   NR_   R`   t   nanRZ   R   i   i    (   R/   R>   R   R0   t   floatRM   t   warnsR2   RK   R3   t   lenR=   (   R,   t   record(    (    sD   lib/python2.7/site-packages/dask/dataframe/tests/test_categorical.pyt   test_categorize_nan  s
    .	c         C   s   t  |  t j ƒ r |  S|  j S(   N(   t
   isinstanceR   RY   R;   (   R   (    (    sD   lib/python2.7/site-packages/dask/dataframe/tests/test_categorical.pyt   get_cat	  s    c         C   s8   t  |  t | t j ƒ r' t j | ƒ n | d | ƒd S(   s@   left and right are equal, treating index and array as equivalentRU   N(   R   R‚   t   npt   ndarrayR   t   Index(   t   leftt   rightRU   (    (    sD   lib/python2.7/site-packages/dask/dataframe/tests/test_categorical.pyt   assert_array_index_eq  s    'c          C   sy   t  j i d d d g d 6ƒ }  |  d j d ƒ |  d <t j |  d ƒ } | j j j ƒ  } t | t j	 j
 ƒ su t ‚ d  S(   NR_   R`   Ra   RZ   R   i   (   R   R0   RL   R/   R>   RZ   R;   t   as_knownR‚   t   coret   SeriesR=   (   R,   t   dask_dft   ret_type(    (    sD   lib/python2.7/site-packages/dask/dataframe/tests/test_categorical.pyt!   test_return_type_known_categories  s
    t   TestCategoricalAccessorc           B   sd  e  Z e j j d  e ƒ e j j d d e f d e f d e f g ƒ d „  ƒ ƒ Z e j j d  e ƒ e j j d d e	 d d	 d
 g ƒ f d i  f d i  f d i  f d e	 d d g ƒ f d e	 d d	 d
 d g ƒ f d e	 d d d d g ƒ f d e	 d d d
 d g ƒ f d i  f g	 ƒ d „  ƒ ƒ Z
 d „  Z e j j d  e ƒ d „  ƒ Z d „  Z d „  Z RS(   t   seriess   prop, compareRb   R   t   codesc         C   sM   | \ } } t  t | ƒ | ƒ } t  t | ƒ | ƒ } | | | d t ƒd  S(   NRU   (   t   getattrRƒ   RJ   (   t   selfR‘   t   propt   comparet   st   dst   expectedt   result(    (    sD   lib/python2.7/site-packages/dask/dataframe/tests/test_categorical.pyt   test_properties  s    s   method, kwargst   add_categoriest   new_categoriest   dt   et
   as_orderedt   as_unorderedt   remove_categoriest   removalsR_   t   rename_categoriest   ft   reorder_categoriesR`   Ra   RG   t   remove_unused_categoriesc   	      C   s©   t  j | |  } | \ } } | t | ƒ ƒ } | t | ƒ ƒ } t | | d t ƒt t | j ƒ j t | ƒ j d t ƒt t | j ƒ j t | ƒ j d t ƒd  S(   NRU   (   t   operatort   methodcallerRƒ   R   RJ   t   _metaRb   R   (	   R”   R‘   t   methodt   kwargst   opR—   R˜   R™   Rš   (    (    sD   lib/python2.7/site-packages/dask/dataframe/tests/test_categorical.pyt   test_callable)  s    !c         C   sP   d „  } d „  } t  j t j | ƒ ƒ  t j | ƒ ƒ  g ƒ } | j j j d  S(   Nc           S   s)   t  j i t  j t j t j g ƒ d 6ƒ S(   NRZ   (   R   R0   R<   R„   R|   (    (    (    sD   lib/python2.7/site-packages/dask/dataframe/tests/test_categorical.pyt
   make_emptyE  s    c           S   s#   t  j i t  j d d g ƒ d 6ƒ S(   NR_   RZ   (   R   R0   R<   (    (    (    sD   lib/python2.7/site-packages/dask/dataframe/tests/test_categorical.pyt	   make_fullH  s    (   R/   t   from_delayedRf   t   delayedRZ   R;   Rb   (   R”   R¯   R°   R_   (    (    sD   lib/python2.7/site-packages/dask/dataframe/tests/test_categorical.pyt   test_categorical_emptyB  s
    		c         C   sT  | \ } } | j  j s t ‚ | j  j ƒ  } | j  j s@ t ‚ t j t ƒ  | j  j Wd  QXt j t ƒ  | j  j Wd  QX| j  j	 d d d g ƒ } | j  j s­ t ‚ t
 j | j  j t | ƒ j ƒ t | j  j t | ƒ j ƒ | j  j ƒ  } | j  j s	t ‚ | j ƒ  } t
 j | j  j t | ƒ j ƒ t | j  j t | ƒ j ƒ d  S(   NR_   R`   Ra   (   R;   RH   R=   t
   as_unknownRM   RN   t   NotImplementedErrorRb   R’   RG   R   R#   Rƒ   R‰   RŠ   R3   (   R”   R‘   R_   t   dat   dbt   res(    (    sD   lib/python2.7/site-packages/dask/dataframe/tests/test_categorical.pyt   test_unknown_categoriesP  s"    c         C   s_   t  j d d d g d d ƒ} t j | d ƒ } | j j ƒ  } | j j ƒ  } t | | ƒ d  S(   NR_   R`   R   R   i   (   R   RŒ   R/   R>   t   strt   upperR   (   R”   R_   R¶   Rš   R™   (    (    sD   lib/python2.7/site-packages/dask/dataframe/tests/test_categorical.pyt   test_categorical_string_opsg  s
    c         C   sW   t  j d d d g d d ƒ} t j | d ƒ } t j t ƒ  | j j ƒ  Wd  QXd  S(   Ni   i   i   R   R   (	   R   RŒ   R/   R>   RM   RN   t   AttributeErrorRº   R»   (   R”   R_   R¶   (    (    sD   lib/python2.7/site-packages/dask/dataframe/tests/test_categorical.pyt"   test_categorical_non_string_raisesn  s    (   t   __name__t
   __module__RM   t   markt   parametrizet
   cat_seriesR‰   R   R›   t   dictR®   R³   R¹   R¼   R¾   (    (    (    sD   lib/python2.7/site-packages/dask/dataframe/tests/test_categorical.pyR     s(   							(    (G   R¨   t   numpyR„   t   pandasR   t   pandas.util.testingt   utilt   testingR   RM   Rf   t   dask.dataframet	   dataframeR/   t   dask.dataframe.coreR    t   dask.dataframe.utilsR   R   R   R   RÃ   RI   RJ   R   RŒ   R<   Rj   R—   R>   R˜   t   appendRp   R3   R   R0   t   arangeR_   R`   Ra   R   R!   R,   R   RL   R   RF   R;   RG   R(   RX   R$   R%   R   R&   R'   R)   R:   R?   RT   R[   RÁ   RÂ   Rn   Rs   R{   R   Rƒ   R‰   R   R   (    (    (    sD   lib/python2.7/site-packages/dask/dataframe/tests/test_categorical.pyt   <module>   sn   "$**#%%..	#			;	$$				