ó
¦–Õ\c           @` s¿	  d  d l  m Z m Z m Z d  d l m Z d  d l Z d  d l Z d  d l m	 Z	 d  d l
 Z
 e
 j d ƒ Z e
 j d ƒ Z d  d l m Z m Z d  d l j j Z d  d l Z d  d l j Z d  d l m Z d  d	 l m Z m Z m Z d  d
 l m Z m  Z  d  d l! m" Z" d  d l# m$ Z$ m% Z% m& Z& m' Z' d  d l( m) Z) m* Z+ m, Z, g  e+ D] Z- e- d f ^ qXg  e, D] Z- e- d f ^ qtZ/ d „  Z0 d „  Z1 d j2 ƒ  Z3 e3 j4 d d ƒ Z5 d j2 ƒ  Z6 d j2 ƒ  Z7 i d d 6d d 6d d 6Z8 d „  e8 j9 ƒ  Dƒ Z: i d d 6d d 6d d 6Z; e j< g  e= e8 ƒ D] Z> e j? e e8 e> ƒ ƒ ^ q4ƒ Z@ d  ZA d! ZB eB j4 d d ƒ ZC e
 jD jE d" ƒ ZF e
 jD jG d# e j? e8 f e
 jH e jI e: d$ eF ƒe jJ e; f g ƒ ZK eK d% „  ƒ ZL eK d& „  ƒ ZM eK d' „  ƒ ZN eK d( „  ƒ ZO eK d) „  ƒ ZP eK d* „  ƒ ZQ eK d+ „  ƒ ZR e
 jD jG d, e j? e j? e8 f e jI e jI e: f g ƒ eF d- „  ƒ ƒ ZS e
 jD jG d. e j? e j? e8 eB f e jI e jI e: eC f g ƒ eF d/ „  ƒ ƒ ZT d0 d1 g d0 d1 g g ZU d2 d3 g d2 d3 g g ZV e
 jD jG d4 e j? eU f e jI eV f g ƒ d5 „  ƒ ZW e
 jD jG d4 e j? eU f e jI eV f g ƒ d6 „  ƒ ZX e
 jD jG d7 e j? e j? e3 d f e jI e jI e5 d f e jI e jI e6 d8 f g ƒ eF d9 „  ƒ ƒ ZY e
 jD jG d: e j? e j? e3 d; f e jI e jI e5 d< d= g f g ƒ eF d> „  ƒ ƒ ZZ e
 jD jG d: e j? e j? e3 d; f e jI e jI e5 d< d? g f g ƒ eF d@ „  ƒ ƒ Z[ e
 jD jG d, e j? e j? e8 f e jI e jI e: f g ƒ eF dA „  ƒ ƒ Z\ e
 jD jG d, e j? e j? e8 f e jI e jI e: f g ƒ eF dB „  ƒ ƒ Z] e
 jD jG dC e j? e8 f e jI e: f g ƒ eF dD „  ƒ ƒ Z^ e
 jD jG dC e j? e8 f e jI e: f g ƒ eF dE „  ƒ ƒ Z_ e
 jD jG dC e j? e8 f e jI e: f g ƒ eF dF „  ƒ ƒ Z` e
 jD jG dC e j? e8 f e jI e: f g ƒ eF dG „  ƒ ƒ Za dH „  Zb dI „  Zc dJ „  Zd dK „  Ze dL „  Zf dM „  Zg dN „  Zh dO „  Zi dP „  Zj e
 jD jk dQ „  ƒ Zl dR „  Zm dS „  Zn dT „  Zo e
 jD jG dU e/ ƒ dV „  ƒ Zp dW „  Zq dX „  Zr dY „  Zs dZ „  Zt d[ „  Zu d\ „  Zv d] „  Zw d^ „  Zx d_ „  Zy d` „  Zz da „  Z{ e
 jD jk db „  ƒ Z| dc „  Z} dd „  Z~ de „  Z df „  Z€ dg „  Z e
 jD jG dh di dj dk g ƒ dl „  ƒ Z‚ dm „  Zƒ dn „  Z„ do „  Z… dp „  Z† dq „  Z‡ dr „  Zˆ ds „  Z‰ dt „  ZŠ du „  Z‹ dv „  ZŒ dw „  Z e
 jD jŽ dx dy ƒ dz „  ƒ Z d{ „  Z d| „  Z‘ d} „  Z’ d~ „  Z“ e
 jD jG d e” d€ f e• d f g ƒ d‚ „  ƒ Z– e
 jD jG dƒ e” e” d„ d… f e• e” d d f e” e• d„ d… f e• e• d d… f d† d‡ g e” dˆ dˆ f d† d‡ g e• dˆ d… f g ƒ d‰ „  ƒ Z— d S(Š   i    (   t   print_functiont   divisiont   absolute_import(   t   BytesION(   t   sleept   pandass   dask.dataframe(   t   partition_allt   valmap(   t   compute_as_if_collection(   t   text_blocks_to_pandast   pandas_read_textt   auto_blocksize(   t	   assert_eqt   has_known_categories(   t
   read_bytes(   t	   filetextst   filetextt   tmpfilet   tmpdir(   t   compresst   filest   seekable_filesi
   c         C` s(   d j  t t j |  j ƒ  j d ƒ ƒ ƒ S(   Ns   
(   t   joint   mapt   strt   stript   split(   t   s(    (    s?   lib/python2.7/site-packages/dask/dataframe/io/tests/test_csv.pyt   normalize_text   s    c         C` s   t  j j |  ƒ d S(   Ni   (   t   ost   pathR   (   R   (    (    s?   lib/python2.7/site-packages/dask/dataframe/io/tests/test_csv.pyt   parse_filename    s    s‘   
name,amount
Alice,100
Bob,-200
Charlie,300
Dennis,400
Edith,-500
Frank,600
Alice,200
Frank,-200
Bob,600
Alice,400
Frank,200
Alice,300
Edith,600
t   ,s   	s¸   
name   amount
Alice    100
Bob     -200
Charlie  300
Dennis   400
Edith   -500
Frank    600
Alice    200
Frank   -200
Bob      600
Alice    400
Frank    200
Alice    300
Edith    600
sÒ  
Date,Open,High,Low,Close,Volume,Adj Close
2015-08-28,198.50,199.839996,197.919998,199.240005,143298900,199.240005
2015-08-27,197.020004,199.419998,195.210007,199.160004,266244700,199.160004
2015-08-26,192.080002,194.789993,188.369995,194.679993,328058100,194.679993
2015-08-25,195.429993,195.449997,186.919998,187.229996,353966700,187.229996
2015-08-24,197.630005,197.630005,182.399994,189.550003,478672400,189.550003
2015-08-21,201.729996,203.940002,197.520004,197.630005,328271500,197.630005
2015-08-20,206.509995,208.289993,203.899994,204.009995,185865600,204.009995
2015-08-19,209.089996,210.009995,207.350006,208.279999,167316300,208.279999
2015-08-18,210.259995,210.679993,209.699997,209.929993,70043800,209.929993
s3   name,amount,id
Alice,100,1
Bob,200,2
Charlie,300,3
s   2014-01-01.csvs   name,amount,id
s   2014-01-02.csvs4   name,amount,id
Dennis,400,4
Edith,500,5
Frank,600,6
s   2014-01-03.csvc         C` s+   i  |  ]! \ } } | j  d  d ƒ | “ q S(   R    s   	(   t   replace(   t   .0t   kt   v(    (    s?   lib/python2.7/site-packages/dask/dataframe/io/tests/test_csv.pys
   <dictcomp>_   s   	 sT       name  amount  id
   Alice     100   1
     Bob     200   2
 Charlie     300   3
s       name  amount  id
sT       name  amount  id
  Dennis     400   4
   Edith     500   5
   Frank     600   6
sL   # some header lines
# that may be present
# in a data file
# before any datas   str, int, int
s   ignore:read_table:FutureWarnings   reader,filest   marksc         C` sz   | d } t  |  | d i  ƒ } t | j ƒ d d d g k sC t ‚ t | ƒ d k s[ t ‚ | j j ƒ  d
 k sv t ‚ d  S(   Ns   2014-01-01.csvt    t   namet   amountt   idi   i   i   i   i   (   R
   t   listt   columnst   AssertionErrort   lenR)   t   sum(   t   readerR   t   bt   df(    (    s?   lib/python2.7/site-packages/dask/dataframe/io/tests/test_csv.pyt   test_pandas_read_textƒ   s
    
$c         C` sQ   | d } t  |  | d i d d g d 6ƒ } t | j ƒ d d g k sM t ‚ d  S(   Ns   2014-01-01.csvR&   R'   R)   t   usecols(   R
   R*   R+   R,   (   R/   R   R0   R1   (    (    s?   lib/python2.7/site-packages/dask/dataframe/io/tests/test_csv.pyt   test_pandas_read_text_kwargsŒ   s    
"c         C` sE   | d } t  |  | d i  i d d 6ƒ } | j j d k sA t ‚ d  S(   Ns   2014-01-01.csvR&   t   floatR(   (   R
   R(   t   dtypeR,   (   R/   R   R0   R1   (    (    s?   lib/python2.7/site-packages/dask/dataframe/io/tests/test_csv.pyt$   test_pandas_read_text_dtype_coercion“   s    
c         C` sœ   | d } | j  d d ƒ \ } } | d } t |  | | i  ƒ } t | j ƒ d d d g k se t ‚ t | ƒ d k s} t ‚ | j j ƒ  d
 k s˜ t ‚ d  S(   Ns   2014-01-01.csvs   
i   R'   R(   R)   i   i   i   i   (   R   R
   R*   R+   R,   R-   R)   R.   (   R/   R   R0   t   headerR1   (    (    s?   lib/python2.7/site-packages/dask/dataframe/io/tests/test_csv.pyt!   test_pandas_read_text_with_headerš   s    

$c   	      C` s;  g  t  | ƒ D] } | | g ^ q } i  } t |  | d d i  ƒ } | d j d ƒ d d } t |  | | | | d t ƒ} t | t j ƒ s– t ‚ t	 | j
 ƒ d d d g k sº t ‚ t |  | | | | d t ƒ} t | t	 ƒ sí t ‚ t | ƒ d	 k st ‚ t d
 „  | Dƒ ƒ s!t ‚ t | j j ƒ  d ƒ d  S(   Ns   2014-01-01.csvR&   s   
i    t
   collectionR'   R(   R)   i   c         s` s   |  ] } t  | d  ƒ Vq d S(   t   daskN(   t   hasattr(   R"   t   item(    (    s?   lib/python2.7/site-packages/dask/dataframe/io/tests/test_csv.pys	   <genexpr>µ   s    id   iÈ   i,  i  iô  iX  i,  iX  iè  iÜ  i4  (   t   sortedR
   R   R	   t   Truet
   isinstancet   ddt	   DataFrameR,   R*   R+   t   FalseR-   t   allR   R(   R.   (	   R/   R   R#   t   blockst   kwargst   headR8   R1   t   values(    (    s?   lib/python2.7/site-packages/dask/dataframe/io/tests/test_csv.pyt!   test_text_blocks_to_pandas_simple¥   s    &	$	c   
      C` só   g  t  | ƒ D] } | | ^ q } g  | D] } | g ^ q* } i d d g d 6} t |  | d d | ƒ } | d j d ƒ d d } t |  | | | | d t ƒ} t | j ƒ d d g k sÅ t ‚ | j ƒ  }	 |	 j | j k j	 ƒ  sï t ‚ d  S(	   NR'   R)   R3   s   2014-01-01.csvR&   s   
i    R:   (
   R>   R
   R   R	   R?   R*   R+   R,   t   computeRD   (
   R/   R   R#   RE   R0   RF   RG   R8   R1   t   result(    (    s?   lib/python2.7/site-packages/dask/dataframe/io/tests/test_csv.pyt!   test_text_blocks_to_pandas_kwargs»   s    #	!c   
      C` sB  | d j  d ƒ d d } g  } x_ t | ƒ D]Q } | | } | j  d ƒ } | j g  t d | ƒ D] } d j | ƒ ^ qc ƒ q. Wt |  | | t j ƒ  i  ƒ } t | j	 ƒ  j
 d t ƒ t j
 d t ƒ d t ƒt d d g }	 t |  | | |	 j ƒ  i d d g d	 6ƒ } t | j	 ƒ  j
 d t ƒ |	 j
 d t ƒ d t ƒd  S(
   Ns   2014-01-01.csvs   
i    i   t   dropt   check_dtypeR'   R)   R3   (   R   R>   t   appendR   R   R	   t   expectedRG   R   RJ   t   reset_indexR?   RC   (
   R/   R   R8   RE   R#   R0   t   linest   bsR1   t	   expected2(    (    s?   lib/python2.7/site-packages/dask/dataframe/io/tests/test_csv.pyt"   test_text_blocks_to_pandas_blockedÊ   s    
6s   dd_read,pd_read,filesc         C` sž   d „  | j  ƒ  Dƒ } t t j ƒ  ƒ } t | d d ƒ^ |  d d | ƒ} t j g  t | ƒ D] } | | d | ƒ^ q` ƒ } t | | d t	 ƒWd  QXd  S(   Nc         S` s'   i  |  ] \ } } t  d  | | “ q S(   s   
(   t   comment_header(   R"   R'   t   content(    (    s?   lib/python2.7/site-packages/dask/dataframe/io/tests/test_csv.pys
   <dictcomp>ã   s   	 t   modeR0   s   2014-01-*.csvt   skiprowsRN   (
   t   itemsR-   RV   t
   splitlinesR   t   pdt   concatR>   R   RC   (   t   dd_readt   pd_readR   t   skipR1   t   nt   expected_df(    (    s?   lib/python2.7/site-packages/dask/dataframe/io/tests/test_csv.pyt   test_skiprowsÞ   s    4s   dd_read,pd_read,files,unitsc         ` s§   ‡  f d †  | j  ƒ  Dƒ } d d d d d g } t | d d ƒ^ |  d	 d
 | ƒ} t j g  t | ƒ D] } | | d
 | ƒ^ qi ƒ } t | | d t ƒWd  QXd  S(   Nc         ` s:   i  |  ]0 \ } } t  d  | j d  d  ˆ  d ƒ | “ q S(   s   
i   (   RV   R!   (   R"   R'   RW   (   t   units(    s?   lib/python2.7/site-packages/dask/dataframe/io/tests/test_csv.pys
   <dictcomp>ð   s   	i    i   i   i   i   RX   R0   s   2014-01-*.csvRY   RN   (   RZ   R   R\   R]   R>   R   RC   (   R^   R_   R   Rd   R`   R1   Ra   Rb   (    (   Rd   s?   lib/python2.7/site-packages/dask/dataframe/io/tests/test_csv.pyt   test_skiprows_as_listë   s    4s   aa,bb
1,1.0
2,2.0s   10,20
30,40s   aa	bb
1	1.0
2	2.0s   10	20
30	40s   reader,blocksc         ` s˜   |  t  | d d ƒ d d ƒ‰  | d d j d ƒ d d } t |  | | ˆ  i  d t ƒ} t j d d | Œ } t ‡  f d †  | Dƒ ƒ s” t ‚ d  S(   Ni    R8   s   
R:   t	   schedulert   syncc         3` s-   |  ]# } | j  j ƒ  ˆ  j  j ƒ  k Vq d  S(   N(   t   dtypest   to_dict(   R"   R1   (   RG   (    s?   lib/python2.7/site-packages/dask/dataframe/io/tests/test_csv.pys	   <genexpr>  s    (   R   R   R	   RC   R;   RJ   RD   R,   (   R/   RE   R8   t   dfs(    (   RG   s?   lib/python2.7/site-packages/dask/dataframe/io/tests/test_csv.pyt   test_enforce_dtypes   s     	c         C` sÂ   | d | d d j  d d ƒ | d d g g } |  t | d d ƒ d d ƒ} | d d j d ƒ d d } t j t ƒ < t |  | | | i  d t d t ƒ} t	 j
 d	 d
 | Œ Wd  QXd  S(   Ni    i   t   at   AR8   s   
R:   t   enforceRf   Rg   (   R!   R   R   t   pytestt   raisest
   ValueErrorR	   RC   R?   R;   RJ   (   R/   RE   RG   R8   Rj   (    (    s?   lib/python2.7/site-packages/dask/dataframe/io/tests/test_csv.pyt   test_enforce_columns  s    2 s   dd_read,pd_read,text,seps   \s+c         C` s’   t  | ƒ € } |  | d d d t j d | ƒ} t | j ƒ d d g k sQ t ‚ | j d d ƒ j d	 t ƒ } t	 | | | d | ƒƒ Wd  QXd  S(
   Nt	   blocksizei   t   lineterminatort   sepR'   R(   Rf   Rg   RM   (
   R   R   t   linesepR*   R+   R,   RJ   RQ   R?   R   (   R^   R_   t   textRu   t   fnt   fRK   (    (    s?   lib/python2.7/site-packages/dask/dataframe/io/tests/test_csv.pyt   test_read_csv  s
    !!s   dd_read,pd_read,text,skipi   i   i   c         C` s\   d d g } t  | ƒ > } |  | d | d | ƒ} t | | | d | d | ƒƒ Wd  QXd  S(   NR'   R(   RY   t   names(   R   R   (   R^   R_   Rw   R`   R{   Rx   t   actual(    (    s?   lib/python2.7/site-packages/dask/dataframe/io/tests/test_csv.pyt   test_read_csv_large_skiprows+  s    i   c         C` sÌ   d d g } t  | ƒ ® } t j t d d ƒH |  | d d d | d | ƒj ƒ  } t | | | d | d | ƒƒ Wd  QXt j t ƒ 7 t j t ƒ ! |  | d d	 d | d | ƒWd  QXWd  QXWd  QXd  S(
   NR'   R(   t   matchs   sample=blocksizeRs   iÈ   RY   R{   i   (   R   Ro   t   warnst   UserWarningRJ   R   Rp   Rq   (   R^   R_   Rw   R`   R{   Rx   R|   (    (    s?   lib/python2.7/site-packages/dask/dataframe/io/tests/test_csv.pyt.   test_read_csv_skiprows_only_in_first_partition6  s    $%c      	   C` sw   t  | d d ƒ_ |  d ƒ } t | t d t ƒd } |  | ƒ } | t | | ƒ ƒ } t | | d t ƒWd  QXd  S(   NRX   R0   s   2014-01-*.csvRN   s   2014-01-01.csv(   R   R   RP   RC   R   (   R^   R_   R   R1   Rx   RT   (    (    s?   lib/python2.7/site-packages/dask/dataframe/io/tests/test_csv.pyt   test_read_csv_filesG  s    c         C` sž   t  | d d ƒ† t | ƒ d  } t j g  | D] } | t | | ƒ ƒ ^ q0 ƒ } |  | ƒ } t | | d t ƒt j t	 ƒ  |  g  ƒ Wd  QXWd  QXd  S(   NRX   R0   i   RN   (
   R   R>   R\   R]   R   R   RC   Ro   Rp   Rq   (   R^   R_   R   t   subsetR#   t   solt   res(    (    s?   lib/python2.7/site-packages/dask/dataframe/io/tests/test_csv.pyt   test_read_csv_files_listV  s    2s   dd_read,filesc         C` s‡   t  | d d ƒo |  d d t d i t d 6ƒ} | j j ƒ  j ƒ  } d | k sY t ‚ d | k sk t ‚ d	 | k s} t ‚ Wd  QXd  S(
   NRX   R0   s   2014-01-*.csvt   include_path_columnt
   convertersR   s   2014-01-01.csvs   2014-01-02.csvs   2014-01-03.csv(   R   R?   R   R   RJ   t   uniqueR,   (   R^   R   R1   t	   filenames(    (    s?   lib/python2.7/site-packages/dask/dataframe/io/tests/test_csv.pyt!   test_read_csv_include_path_columne  s    c         C` s‡   t  | d d ƒo |  d d d d i t d 6ƒ} | j j ƒ  j ƒ  } d | k sY t ‚ d | k sk t ‚ d	 | k s} t ‚ Wd  QXd  S(
   NRX   R0   s   2014-01-*.csvR‡   t   filenameRˆ   s   2014-01-01.csvs   2014-01-02.csvs   2014-01-03.csv(   R   R   RŒ   RJ   R‰   R,   (   R^   R   R1   RŠ   (    (    s?   lib/python2.7/site-packages/dask/dataframe/io/tests/test_csv.pyt(   test_read_csv_include_path_column_as_strs  s    c         C` sC   t  | d d ƒ+ t j t ƒ  |  d d d ƒWd  QXWd  QXd  S(   NRX   R0   s   2014-01-*.csvR‡   R'   (   R   Ro   Rp   Rq   (   R^   R   (    (    s?   lib/python2.7/site-packages/dask/dataframe/io/tests/test_csv.pyt5   test_read_csv_include_path_column_with_duplicate_name  s    c      
   C` s±   t  | d d ƒ™ |  d d t ƒ} | j j d k s= t ‚ t | j ƒ sR t ‚ |  d d t d t ƒ} | d j ƒ  } | j j d k s’ t ‚ t | j ƒ s§ t ‚ Wd  QXd  S(   NRX   R0   s   2014-01-*.csvR‡   t   categoryR:   i    (   R   R?   R   R6   R,   R   RC   RJ   (   R^   R   R1   Rj   RK   (    (    s?   lib/python2.7/site-packages/dask/dataframe/io/tests/test_csv.pyt3   test_read_csv_include_path_column_is_dtype_category‹  s    c       
   C` s:  t  t ƒ (}  t j |  d d ƒj d ƒ } | j d d ƒ } | j j d k sW t ‚ t	 t j
 | j | j ƒ  d d ƒ} x t | ƒ D] \ } } | t | j ƒ d k  rÖ | j | j | d k  j ƒ  sÖ t ‚ n  | d k rˆ | j | j | k j ƒ  st ‚ qˆ qˆ Wt j |  ƒ j d ƒ } t | | ƒ Wd  QXd  S(	   NRs   i   R(   Rf   Rg   i   i   i    (   R   t   csv_textRA   t   read_csvt	   set_indexRJ   t   indexR'   R,   R   RB   R;   t   __dask_keys__t	   enumerateR-   t	   divisionsRD   R\   R   (   Rx   Ry   RK   RE   t   it   blockRP   (    (    s?   lib/python2.7/site-packages/dask/dataframe/io/tests/test_csv.pyt   test_read_csv_indexŸ  s    	))c       	   C` sb   t  t ƒ P }  t j |  d t d ƒ ƒ} | } t j |  d t d ƒ ƒ} t | | ƒ Wd  QXd  S(   NRY   i   (   R   R‘   RA   R’   t   rangeR\   R   (   Rx   Ry   RK   RP   (    (    s?   lib/python2.7/site-packages/dask/dataframe/io/tests/test_csv.pyt   test_read_csv_skiprows_range²  s
    c          C` sy   t  t ƒ g }  t j |  d d d d d g ƒ} t j |  d d d g ƒ} | j ƒ  j | j k j ƒ  so t ‚ Wd  QXd  S(   NRs   i   R3   t   Hight   Low(	   R   t
   timeseriesRA   R’   R\   RJ   RH   RD   R,   (   Rx   R1   RP   (    (    s?   lib/python2.7/site-packages/dask/dataframe/io/tests/test_csv.pyt   test_usecolsº  s    !c          C` s…   t  t ƒ s }  t j |  d d ƒ} t j |  d d ƒ} | j | j k sQ t ‚ t j |  d d ƒ} | j d k s{ t ‚ Wd  QXd  S(   NRs   t   30Bt   30t   64MiBi   (   R   RŸ   RA   R’   t   npartitionsR,   (   Rx   Rl   R0   t   c(    (    s?   lib/python2.7/site-packages/dask/dataframe/io/tests/test_csv.pyt   test_string_blocksizeÁ  s    c       
   C` sv   t  d ƒ }  t |  ƒ X } t j | d t d d ƒ} d | j k sK t ‚ | j j ƒ  j	 ƒ  d k sl t ‚ Wd  QXd  S(   Nsm   
    name, amount
    Alice,100
    Bob,-200
    Charlie,300
    Dennis,400
    Edith,-500
    Frank,600
    t   skipinitialspaceRs   i   R(   iX  (
   R   R   RA   R’   R?   R+   R,   R(   t   maxRJ   (   Rw   Rx   R1   (    (    s?   lib/python2.7/site-packages/dask/dataframe/io/tests/test_csv.pyt   test_skipinitialspaceË  s    	c          C` sX   t  d ƒ }  t |  ƒ : } t j | d d ƒ} | j j ƒ  j t k sN t ‚ Wd  QXd  S(   Nsp   
    name,amount
    Alice,100.5
    Bob,-200.5
    Charlie,300
    Dennis,400
    Edith,-500
    Frank,600
    Rs   i   (	   R   R   RA   R’   R(   RJ   R6   R5   R,   (   Rw   Rx   R1   (    (    s?   lib/python2.7/site-packages/dask/dataframe/io/tests/test_csv.pyt   test_consistent_dtypesÝ  s
    	c          C` sˆ   t  d ƒ }  t  d ƒ } t i |  d 6| d 6ƒ P t j d d d ƒ} | j j t k s` t ‚ | j j ƒ  j t k s~ t ‚ Wd  QXd  S(   Ns@   
    name,amount
    Alice,100
    Bob,-200
    Charlie,300
    s8   
    name,amount
    1,400
    2,-500
    Frank,600
    s	   foo.1.csvs	   foo.2.csvs	   foo.*.csvRs   i   (	   R   R   RA   R’   R'   R6   t   objectR,   RJ   (   t   text1t   text2R1   (    (    s?   lib/python2.7/site-packages/dask/dataframe/io/tests/test_csv.pyt   test_consistent_dtypes_2í  s    		c       
   C` sÞ   t  d ƒ }  t  d ƒ } t i |  d 6| d 6ƒ ¦ t j d d i d d 6d	 d
 ƒ} | j j d k sm t ‚ t | j ƒ sƒ t ‚ | j ƒ  } | j j d k s§ t ‚ t	 | j j
 j ƒ d d d d g k sÔ t ‚ Wd  QXd  S(   NsJ   
    fruit,count
    apple,10
    apple,25
    pear,100
    orange,15
    sO   
    fruit,count
    apple,200
    banana,300
    orange,400
    banana,10
    s	   foo.1.csvs	   foo.2.csvs	   foo.*.csvR6   R   t   fruitRs   i   t   applet   bananat   oranget   pear(   R   R   RA   R’   R¯   R6   R,   R   RJ   R>   t   catt
   categories(   R¬   R­   R1   R…   (    (    s?   lib/python2.7/site-packages/dask/dataframe/io/tests/test_csv.pyt   test_categorical_dtypes  s    		"c          C` s  t  d ƒ }  t  d ƒ } t j j j d d d g ƒ } t i |  d 6| d 6ƒ ­t j d d	 i d
 d 6d
 d 6ƒ} | j j	 j
 t k s t ‚ | j j	 j
 t k sª t ‚ t j i t j d d d d d d g d | j ƒd 6t j d d d d d d g d | j ƒd 6d d d d d d d g ƒ} t | | ƒ t j d d	 i | d 6d
 d 6ƒ} | j j	 j
 t k srt ‚ | j j	 j
 t k st ‚ t j | j j	 j | j ƒ | j j	 j t k sÄt ‚ t | | ƒ t j j j d d d g d t ƒ} t j d d	 i | d 6d
 d 6ƒ} | d j	 j ƒ  | d <| j j	 j
 t k sJt ‚ | j j	 j
 t k set ‚ | j j	 j t k s€t ‚ t | | ƒ t j d d	 t j j j ƒ  ƒ} | j j	 j
 t k sÉt ‚ t j d d	 d
 ƒ} | j j	 j
 t k sùt ‚ Wd  QXd  S(   Ns%   
    A,B
    a,a
    b,b
    a,a
    s%   
    A,B
    a,a
    b,b
    c,c
    Rl   R0   R¥   s	   foo.1.csvs	   foo.2.csvs	   foo.*.csvR6   R   Rm   t   BRµ   R”   i    i   i   t   ordered(   R   R\   t   apit   typest   CategoricalDtypeR   RA   R’   Rm   R´   t   knownRC   R,   R·   RB   t   CategoricalRµ   R   R?   t   tmt   assert_index_equalR¸   t
   as_ordered(   R¬   R­   R6   RK   RP   (    (    s?   lib/python2.7/site-packages/dask/dataframe/io/tests/test_csv.pyt   test_categorical_known  sF    			#$#c          C` s  t  ƒ  ò }  t j t j j |  d ƒ d ƒ } | j t j ƒ  ƒ | j	 ƒ  t j t j j |  d ƒ d ƒ } | j t j ƒ  ƒ | j	 ƒ  t
 j t ƒ ) t j t j j |  d ƒ d d ƒ} Wd  QXt | j ƒ  ƒ t t j d ƒ ƒ d d	 k s÷ t ‚ Wd  QXd  S(
   Ns   a.csv.gzt   wbs   b.csv.gzs   *.csv.gzt   compressiont   gzips   
i   i   (   R   RÄ   t   openR   R   R   t   writeR‘   t   encodet   closeR¾   t   assert_produces_warningR€   RA   R’   R-   RJ   R   R,   (   t   tdirRy   R1   (    (    s?   lib/python2.7/site-packages/dask/dataframe/io/tests/test_csv.pyt   test_compression_multiple_filesS  s    !
!
c          C` sm   t  d ƒ [ }  t j |  d d ƒ} t | j ƒ  ƒ d k sB t ‚ t | j ƒ d d g k sc t ‚ Wd  QXd  S(   Ns   a,bR8   i    Rl   R0   (   R   RA   R’   R-   RJ   R,   R*   R+   (   Rx   R1   (    (    s?   lib/python2.7/site-packages/dask/dataframe/io/tests/test_csv.pyt   test_empty_csv_filee  s    c          C` sX   t  t d d ƒ@ }  t j |  d t ƒ} t | j ƒ d d d g k sN t ‚ Wd  QXd  S(   NRX   R0   t   sampleR'   R(   R)   (   R   t	   csv_filesRA   R’   RC   R*   R+   R,   (   Rx   R1   (    (    s?   lib/python2.7/site-packages/dask/dataframe/io/tests/test_csv.pyt   test_read_csv_no_samplel  s    c          C` s_   t  t d d ƒG t j d d t ƒ}  t j d d t ƒ} |  j | j k sU t ‚ Wd  QXd  S(   NRX   R0   s   2014-01-*.csvRn   (   R   RÎ   RA   R’   R?   RC   t   _nameR,   (   Rl   R0   (    (    s?   lib/python2.7/site-packages/dask/dataframe/io/tests/test_csv.pyt"   test_read_csv_sensitive_to_enforcer  s    s   fmt,blocksizec      
   C` s‚   t  t |  t ƒ } t | d d ƒW t j d d |  d | ƒ} t | j d d ƒ j d t	 ƒ t
 j d t	 ƒ d	 t ƒWd  QXd  S(
   NRX   R0   s   2014-01-*.csvRÃ   Rs   Rf   Rg   RM   RN   (   R   R   RÎ   R   RA   R’   R   RJ   RQ   R?   RP   RC   (   t   fmtRs   t   files2R1   (    (    s?   lib/python2.7/site-packages/dask/dataframe/io/tests/test_csv.pyt   test_read_csv_compressiony  s
    c          C` sM  t  t d t ƒ }  t |  d d ƒ"t j t ƒ 1 } t j d d d ƒ} | j	 d k sb t
 ‚ Wd  QXt | ƒ d k s€ t
 ‚ t | d j ƒ } d | k s¥ t
 ‚ d	 | k s· t
 ‚ t j d  ƒ " } t j d d d d
 d  ƒ} Wd  QXt | ƒ d k st
 ‚ t j t ƒ 0 t j t ƒ  t j d d d ƒ} Wd  QXWd  QXWd  QXd  S(   NRÄ   RX   R0   s   2014-01-*.csvRÃ   i   i   i    s   blocksize=NoneRs   t   foo(   R   R   RÎ   R   Ro   R   R€   RA   R’   R¤   R,   R-   R   t   messaget   NoneRp   t   NotImplementedError(   RÓ   t   wR1   t   msg(    (    s?   lib/python2.7/site-packages/dask/dataframe/io/tests/test_csv.pyt   test_warn_non_seekable_files‚  s     c       
   C` s|   d }  t  |  ƒ d } t j | d d d d ƒ} | j j ƒ  j ƒ  d k sQ t ‚ | j j ƒ  j ƒ  d k sr t ‚ Wd  QXd  S(   Ns!   a,b
1,2
2,3
3,4
4,5
5,6
6,7Rs   i   Rt   s   
i   i   i   i   i   i   i   i	   i   i   i   i   i   i
   i   i   (   R   RA   R’   R0   R.   RJ   R,   Rl   (   Rw   Rx   R1   (    (    s?   lib/python2.7/site-packages/dask/dataframe/io/tests/test_csv.pyt   test_windows_line_terminator™  s
    !c          C` s‰   t  i d d 6d d 6d d 6ƒ b t j d d d  ƒ}  t j i d	 d
 g d 6d d g d	 6ƒ } t |  j ƒ  j d t	 ƒ | ƒ Wd  QXd  S(   Ns   1,2s
   .tmp.1.csvR&   s
   .tmp.2.csvs   3,4s
   .tmp.3.csvs
   .tmp.*.csvR8   i   i   i    i   i   RM   (
   R   RA   R’   R×   R\   RB   R   RJ   RQ   R?   (   R1   RP   (    (    s?   lib/python2.7/site-packages/dask/dataframe/io/tests/test_csv.pyt   test_header_None¡  s    )c           C` sX   t  t d d ƒ t ƒ s t ‚ t d d ƒ d k s9 t ‚ t d d ƒ d k sT t ‚ d  S(   Ni¸  i   i   id   iˆ  i   iú   (   R@   R   t   intR,   (    (    (    s?   lib/python2.7/site-packages/dask/dataframe/io/tests/test_csv.pyt   test_auto_blocksizeª  s    c          C` s@   t  d d ƒ }  |  t d ƒ k s' t ‚ t |  t ƒ s< t ‚ d  S(   NI ¥Ôè   i   g    €„ŽA(   R   RÞ   R,   R@   (   Rs   (    (    s?   lib/python2.7/site-packages/dask/dataframe/io/tests/test_csv.pyt   test_auto_blocksize_max64mb°  s    c         C` sô   t  j d ƒ } y d d l m } Wn  t k
 rE t  j d ƒ } n X| j ƒ  j } | j ƒ  } | j d t	 ƒ } |  j
 t j j j d | ƒ t | | ƒ } t t d d ƒ> t j d	 ƒ | j sÍ t ‚ | j d
 d | k sê t ‚ Wd  QXd  S(   Nt   psutili    (   t   mockRâ   t   wrapsR   RX   R0   s   2014-01-01.csvi   Rs   (   Ro   t   importorskipt   unittestRâ   t   ImportErrort   virtual_memoryt   totalt	   cpu_countt   MockR   t   setattrR;   t	   dataframet   iot   csvR   R   RÎ   RA   R’   t   calledR,   t	   call_args(   t   monkeypatchRá   Râ   t   total_memoryRé   t   mock_read_bytest   expected_block_size(    (    s?   lib/python2.7/site-packages/dask/dataframe/io/tests/test_csv.pyt   test_auto_blocksize_csv¶  s    c          C` sn   i d d 6d d 6}  t  |  ƒ H t j d d d ƒt j d d d ƒ} | j d k j ƒ  sd t ‚ Wd  QXd  S(	   NsA   a,b
0,"abcdefghijklmnopqrstuvwxyz"
1,"abcdefghijklmnopqrstuvwxyz"s   .overflow1.csvs.   a,b
111111,-11111
222222,-22222
333333,-33333
s   .overflow2.csvRÍ   i4   i#   t   i8(   R   RA   R’   Rh   RD   R,   (   R   R1   (    (    s?   lib/python2.7/site-packages/dask/dataframe/io/tests/test_csv.pyt   test_head_partial_line_fixÈ  s    

c          C` s[   d }  y t  j |  ƒ t s" t ‚ Wn1 t t f k
 rV } |  t | ƒ k sW t ‚ n Xd  S(   Ns   .not.a.real.file.csv(   RA   R’   RC   R,   t   OSErrort   IOErrorR   (   Rx   t   e(    (    s?   lib/python2.7/site-packages/dask/dataframe/io/tests/test_csv.pyt    test_read_csv_raises_on_no_filesÛ  s    c       
   C` sÙ   t  t ƒ Ç }  t j |  ƒ } t j |  ƒ } | j | j k sE t ‚ t | j j ƒ  d t	 ƒt | j j ƒ  d t	 ƒk s t ‚ t
 | j t	 ƒ s™ t ‚ t j |  d d d d g ƒ} | j | j k sÏ t ‚ Wd  QXd  S(   Nt   keyRY   i   t	   na_valuesi    (   R   R‘   RA   R’   RÐ   R,   R>   R;   t   keysR   R@   (   Rx   Rl   R0   R¥   (    (    s?   lib/python2.7/site-packages/dask/dataframe/io/tests/test_csv.pyt$   test_read_csv_has_deterministic_nameä  s    <c       	   C` s   t  i t d 6t d 6ƒ _ t j d ƒ }  t j d ƒ } t |  j j ƒ  d t ƒt | j j ƒ  d t ƒk su t ‚ Wd  QXd  S(   Ns
   _foo.1.csvs
   _foo.2.csvs
   _foo.*.csvRü   (	   R   R‘   RA   R’   R>   R;   Rþ   R   R,   (   Rl   R0   (    (    s?   lib/python2.7/site-packages/dask/dataframe/io/tests/test_csv.pyt-   test_multiple_read_csv_has_deterministic_nameð  s    c          C` sO   t  d ƒ = }  t j |  d d  ƒ} t | j ƒ d d g k sE t ‚ Wd  QXd  S(   Ns   alice,1
bob,2R8   i    i   (   R   RA   R’   R×   R*   R+   R,   (   Rx   R1   (    (    s?   lib/python2.7/site-packages/dask/dataframe/io/tests/test_csv.pyt   test_csv_with_integer_namesø  s    c       
   C` s­   t  t ƒ › }  t d ƒ t j |  ƒ } t d ƒ t |  d ƒ ! } | j d ƒ t j | ƒ Wd  QXt j |  ƒ } t	 | j
 d t ƒt	 | j
 d t ƒk s£ t ‚ Wd  QXd  S(   Ni   Rl   s   
George,700Rü   (   R   R‘   R   RA   R’   RÅ   RÆ   R   t   fsyncR>   R;   R   R,   (   Rx   Rl   Ry   R0   (    (    s?   lib/python2.7/site-packages/dask/dataframe/io/tests/test_csv.pyt1   test_read_csv_of_modified_file_has_different_nameþ  s    

c          C` s”  d }  x t  d ƒ D] } |  d 7}  q W|  d 7}  d } t |  ƒ K} t j | ƒ } d } t j t ƒ / } t j | d d d	 d
 g ƒj d d ƒ Wd  QX| j	 | | ƒ s´ t
 ‚ t j t ƒ & } t j | d d ƒj d d ƒ Wd  QX| j	 | ƒ s t
 ‚ d } t j t ƒ 3 } t j | d d d i d d 6ƒj d d ƒ Wd  QXt | j ƒ | k set
 ‚ t j t ƒ < } t j | d d d	 d
 g d i d d 6ƒj d d ƒ Wd  QXt | j ƒ | | k sÑt
 ‚ d } t j t ƒ J } t j | d d d	 d
 g d i t d 6t d 6t d 6ƒj d d ƒ Wd  QXt | j ƒ | k sMt
 ‚ t j | d d d i t d 6t d 6t d 6ƒ} t | | ƒ Wd  QXd  S(   Ns*   numbers,names,more_numbers,integers,dates
iè  s   1,,2,3,2017-10-31 00:00:00
s"   1.5,bar,2.5,3,4998-01-01 00:00:00
s\  

-------------------------------------------------------------

The following columns also failed to properly parse as dates:

- dates

This is usually due to an invalid value in that column. To
diagnose and fix it's recommended to drop these columns from the
`parse_dates` keyword, and manually convert them to dates later
using `dd.to_datetime`.sU  Mismatched dtypes found in `pd.read_csv`/`pd.read_table`.

+--------------+---------+----------+
| Column       | Found   | Expected |
+--------------+---------+----------+
| more_numbers | float64 | int64    |
| names        | object  | float64  |
| numbers      | float64 | int64    |
+--------------+---------+----------+

- names
  ValueError(.*)

Usually this is due to dask's dtype inference failing, and
*may* be fixed by specifying dtypes manually by adding:

dtype={'more_numbers': 'float64',
       'names': 'object',
       'numbers': 'float64'}

to the call to `read_csv`/`read_table`.RÍ   i2   t   parse_datest   datesRf   Rg   sa  Mismatched dtypes found in `pd.read_csv`/`pd.read_table`.

+--------------+---------+----------+
| Column       | Found   | Expected |
+--------------+---------+----------+
| more_numbers | float64 | int64    |
| numbers      | float64 | int64    |
+--------------+---------+----------+

Usually this is due to dask's dtype inference failing, and
*may* be fixed by specifying dtypes manually by adding:

dtype={'more_numbers': 'float64',
       'numbers': 'float64'}

to the call to `read_csv`/`read_table`.

Alternatively, provide `assume_missing=True` to interpret
all unspecified integer columns as floats.R6   t   OR{   sQ  Mismatched dtypes found in `pd.read_csv`/`pd.read_table`.

The following columns failed to properly parse as dates:

- dates

This is usually due to an invalid value in that column. To
diagnose and fix it's recommended to drop these columns from the
`parse_dates` keyword, and manually convert them to dates later
using `dd.to_datetime`.t   more_numberst   numbers(   R›   R   R\   R’   Ro   Rp   Rq   RA   RJ   R~   R,   R   t   valueR5   R«   R   (   Rw   R˜   t   date_msgRx   R„   RÚ   Rú   R…   (    (    s?   lib/python2.7/site-packages/dask/dataframe/io/tests/test_csv.pyt   test_late_dtypes  sD    
%  c          C` s’  d }  x t  d ƒ D] } |  d 7}  q W|  d 7}  t |  ƒ Á } t j | ƒ } t j | d d d t ƒ} t | | j i t d 6ƒ ƒ t j | d d d t d	 i d
 d 6ƒ} t | | ƒ t j | d d d t d	 d  ƒ} t | | j i t d 6ƒ ƒ Wd  QXd }  x t  d ƒ D] } |  d 7}  qW|  d 7}  t |  ƒ O } t j | ƒ } t j | d d d	 d
 d t ƒ} | j
 j d
 k sˆt ‚ Wd  QXd  S(   Ns$   numbers,names,more_numbers,integers
iè  s
   1,foo,2,3
s   1.5,bar,2.5,3
RÍ   i2   t   assume_missingt   integersR6   t   int64s   numbers,integers
s   1,2
s   1.5,2
i   (   R›   R   R\   R’   RA   R?   R   t   astypeR5   R×   R  R6   R,   (   Rw   R˜   Rx   R„   R…   R1   (    (    s?   lib/python2.7/site-packages/dask/dataframe/io/tests/test_csv.pyt   test_assume_missingv  s*    
!#
!c       
   C` sp   t  t ƒ ^ }  y) t j |  d d d d ƒt s7 t ‚ Wn+ t k
 re } d t | ƒ k sf t ‚ n XWd  QXd  S(   NRs   i   t	   index_colR'   R“   (   R   R‘   RA   R’   RC   R,   Rq   R   (   Rx   Rú   (    (    s?   lib/python2.7/site-packages/dask/dataframe/io/tests/test_csv.pyt   test_index_col˜  s    c          C` sÏ   t  t ƒ ½ }  t j |  d d d d d d d g d d g ƒ} t j |  d d d d d g d d g d d	 ƒj d ƒ } t | | ƒ t j |  d d d d d g d d g ƒj d ƒ } t | | ƒ Wd  QXd  S(
   NR  i    R8   R3   i   R  t   DateRs   i€–˜ (   R   RŸ   R\   R’   RA   R“   R   (   Rx   R1   t   ddf(    (    s?   lib/python2.7/site-packages/dask/dataframe/io/tests/test_csv.pyt0   test_read_csv_with_datetime_index_partitions_one¡  s    $	c          C` s   t  t ƒ } }  t j |  d d d d d d d g d d g ƒ} t j |  d d d d d g d d g d d	 ƒj d ƒ } t | | ƒ Wd  QXd  S(
   NR  i    R8   R3   i   R  R  Rs   i  (   R   RŸ   R\   R’   RA   R“   R   (   Rx   R1   R  (    (    s?   lib/python2.7/site-packages/dask/dataframe/io/tests/test_csv.pyt.   test_read_csv_with_datetime_index_partitions_n±  s    $	t   encodings   utf-16s	   utf-16-les	   utf-16-bec   	   
   C` sô   t  j t d d ƒ ƒ } | d } | d } | d } t  j i | d 6| d 6| d 6| d	 6ƒ } t d
 ƒ  } | j | d |  d t ƒt  j | d |  ƒ} t j | d |  d d ƒ} | j	 ƒ  } t t
 | j ƒ ƒ | _ t | | ƒ Wd  QXd  S(   Ni    id   i   gffffff
@g”‡…ZÓ¼ÿ?Rl   R0   R¥   t   ds   .csvR  R”   Rs   iè  (   R\   t   SeriesR›   RB   R   t   to_csvRC   R’   RA   RJ   R-   R”   R   (	   R  t   art   brt   crt   drt   test_dfRx   Rl   R  (    (    s?   lib/python2.7/site-packages/dask/dataframe/io/tests/test_csv.pyt   test_encoding_gh601¼  s    


+c       
   C` s   d j  d d ƒ }  t |  ƒ i } t j | d d ƒ} t | t j | d d ƒƒ t j | d d ƒ} t | t j | d d ƒƒ Wd  QXd  S(   Ns   a b c-d
1 2 3
4 5 6t    s   	Ru   t	   delimiter(   R!   R   RA   R’   R   R\   (   Rw   Rx   R1   (    (    s?   lib/python2.7/site-packages/dask/dataframe/io/tests/test_csv.pyt   test_read_csv_header_issue_823Î  s    c       
   C` sJ   t  t ƒ 8 }  t j |  d d  ƒ} t | t j |  d d  ƒƒ Wd  QXd  S(   NR3   (   R   R‘   RA   R’   R×   R   R\   (   Rx   R1   (    (    s?   lib/python2.7/site-packages/dask/dataframe/io/tests/test_csv.pyt   test_none_usecolsØ  s    c       	   C` s   t  d ƒ }  t |  ƒ  } t j | d d d g g ƒ} t j | d d d g g ƒ} | j | j k j ƒ  su t ‚ t | ƒ t | ƒ k s“ t ‚ Wd  QXd  S(   NsX  
    ID,date,time
    10,2003-11-04,180036
    11,2003-11-05,125640
    12,2003-11-01,2519
    13,2003-10-22,142559
    14,2003-10-24,163113
    15,2003-10-20,170133
    16,2003-11-11,160448
    17,2003-11-03,171759
    18,2003-11-07,190928
    19,2003-10-21,84623
    20,2003-10-25,192207
    21,2003-11-13,180156
    22,2003-11-15,131037
    R  t   datet   time(	   R   R   RA   R’   R\   R+   RD   R,   R-   (   t	   pdmc_textRx   R  R1   (    (    s?   lib/python2.7/site-packages/dask/dataframe/io/tests/test_csv.pyt   test_parse_dates_multi_columnÞ  s    	c       
   C` s—   t  d ƒ }  t |  ƒ y } t j | d d d d ƒ} t j | d d d d ƒ} | j | j k j ƒ  so t ‚ t | ƒ t | ƒ k s t ‚ Wd  QXd  S(   NsB   
    name###amount
    alice###100
    bob###200
    charlie###300Ru   s   ###t   enginet   python(	   R   R   RA   R’   R\   R+   RD   R,   R-   (   t   sep_textRx   R  R1   (    (    s?   lib/python2.7/site-packages/dask/dataframe/io/tests/test_csv.pyt   test_read_csv_sepø  s    	c          C` sj   d d d }  t  |  d d ƒD } t j | d d  d d d	 d
 d d d g d d ƒj d d ƒ Wd  QXd  S(   Ns   0,my
1,data
iè  s	   2,foobarRX   RÂ   R8   Ru   R    Rt   s   
R{   Rl   R0   Rs   iÈ   Rf   Rg   (   R   RA   R’   R×   RJ   (   t   dataRx   (    (    s?   lib/python2.7/site-packages/dask/dataframe/io/tests/test_csv.pyt   test_read_csv_slash_r  s    c       
   C` sP   d }  t  |  d d ƒ2 } t t j | d t ƒt j | d t ƒƒ Wd  QXd  S(   Ns   a,b
1,2
3,4
5,6RX   RÂ   R6   (   R   R   R\   R’   R5   RA   (   R-  Rx   (    (    s?   lib/python2.7/site-packages/dask/dataframe/io/tests/test_csv.pyt   test_read_csv_singleton_dtype  s    c          C` sœ   t  j ƒ  }  t |  ƒ d } |  | j d d ƒ |  | <t |  d d ƒN t j d ƒ } t j d ƒ } | j | j k j	 ƒ  s… t
 ‚ t | | ƒ Wd  QXd  S(   NiÿÿÿÿR'   t   NameRX   R0   s   2014-01-*.csvs   2014-01-01.csv(   RÎ   t   copyR>   R!   R   RA   R’   R\   R+   RD   R,   R   (   R   R#   R  R1   (    (    s?   lib/python2.7/site-packages/dask/dataframe/io/tests/test_csv.pyt   test_robust_column_mismatch  s    c          C` s   d }  t  |  ƒ g } d } t j t ƒ  t j | d | ƒWd  QXt t j | d | d d  ƒt j | d d  ƒƒ Wd  QXd } | |  }  t  |  ƒ ƒ } d t	 | ƒ } t j t ƒ  t j | d | d d ƒWd  QXt t j | d | d d  d d ƒt j | d d  d d ƒƒ Wd  QXd  S(   NsA   AAAAA,BBBBB,CCCCC,DDDDD,EEEEE
1,2,3,4,5
6,7,8,9,10
11,12,13,14,15i   RÍ   R8   s   # skip
# these
# lines
RY   i   (
   R   Ro   Rp   Rq   RA   R’   R   R×   R\   R-   (   Rw   Rx   RÍ   t   skiptext(    (    s?   lib/python2.7/site-packages/dask/dataframe/io/tests/test_csv.pyt!   test_error_if_sample_is_too_small   s    
!c       
   C` sn   d }  d d g } t  |  ƒ J } t j | d | d d ƒ} t j | d | ƒ} t | | d t ƒWd  QXd  S(   Ns?   Alice,100
Bob,-200
Charlie,300
Dennis,400
Edith,-500
Frank,600
R'   R(   R{   Rs   i   t   check_index(   R   RA   R’   R\   R   RC   (   Rw   R{   Rx   R  R1   (    (    s?   lib/python2.7/site-packages/dask/dataframe/io/tests/test_csv.pyt   test_read_csv_names_not_none?  s    c          C` s©  t  j i d d d d g d 6d d d d	 g d
 6ƒ }  xmd d g D]_} t j |  | ƒ } t ƒ  W } | j | d t ƒt j t j	 j
 | d ƒ ƒ j ƒ  j d t ƒ } t | |  ƒ Wd  QXt ƒ  r } | j | d t d t ƒ} t j d d | Œ t j t j	 j
 | d ƒ ƒ j ƒ  j d t ƒ } t | |  ƒ Wd  QXt ƒ  ] } t j	 j
 | d ƒ } | j | d t ƒt j | ƒ j ƒ  j d t ƒ } t | |  ƒ Wd  QXqB Wd  S(   NRl   R0   R¥   R  t   xi   i   i   i   t   yR”   t   *RM   RJ   Rf   Rg   s
   data_*.csv(   R\   RB   RA   t   from_pandasR   R  RC   R’   R   R   R   RJ   RQ   R?   R   R;   (   R1   R¤   Rl   t   dnRK   t   rRx   (    (    s?   lib/python2.7/site-packages/dask/dataframe/io/tests/test_csv.pyt   test_to_csvQ  s$    00!c          C` s²  t  j i d d d d g d 6d d d d	 g d
 6ƒ }  t j |  d ƒ } t ƒ  ? } t j t ƒ ' t j	 j
 | d ƒ } | j | ƒ Wd  QXWd  QXt  j i d d d d d d d d d d d d d d d d g d 6d d d d	 d d d d d d d d d  d! d" d# g d
 6ƒ } t j | d# ƒ } t ƒ  ] } t j	 j
 | d$ ƒ } | j | d% t ƒt j | ƒ j ƒ  j d& t ƒ } t | | ƒ Wd  QXt j |  d ƒ } t ƒ  v } | j | d% t ƒt j	 j
 | d$ ƒ } | j | d' d( d% t ƒt j | ƒ j ƒ  j d& t ƒ } t | |  ƒ Wd  QXt j | d# ƒ } t ƒ  v } | j | d% t ƒt j	 j
 | d$ ƒ } | j | d' d( d% t ƒt j | ƒ j ƒ  j d& t ƒ } t | | ƒ Wd  QXd  S()   NRl   R0   R¥   R  R7  i   i   i   i   R8  s   data_*_*.csvRú   Ry   t   gt   hR˜   t   jR#   t   lt   mRa   t   ot   pi   i   i   i   i	   i
   i   i   i   i   i   i   s
   data_*.csvR”   RM   RX   RÙ   (   R\   RB   RA   R:  R   Ro   Rp   Rq   R   R   R   R  RC   R’   RJ   RQ   R?   R   (   R1   Rl   R;  Rx   t   df16RK   (    (    s?   lib/python2.7/site-packages/dask/dataframe/io/tests/test_csv.pyt&   test_to_csv_multiple_files_cornercasesi  s>    !"!!!t   reasons#   to_csv does not support compressionc          C` sÇ   t  j i d d d d g d 6d d d d	 g d
 6d d d d d g ƒ}  xy d d g D]k } t j |  | ƒ } t d ƒ E } | j | d d ƒt  j | d d d d ƒ} t j | |  ƒ Wd  QXqT Wd  S(   NRl   R0   R¥   R  R7  i   i   i   i   R8  R”   g      ð?g       @g      @g      @Rî   RÃ   RÄ   R  i    (	   R\   RB   RA   R:  R   R  R’   R¾   t   assert_frame_equal(   R1   R¤   Rl   Rx   RK   (    (    s?   lib/python2.7/site-packages/dask/dataframe/io/tests/test_csv.pyt   test_to_csv_gzip  s    +c          C` sè   t  j i d d d d g d 6d d d d	 g d
 6d d d d d g ƒ}  t j |  d d ƒ} t ƒ  Y } t | ƒ } | j | ƒ t j | ƒ s– t	 ‚ t j
 t j j | d ƒ ƒ j ƒ  } Wd  QX| j j |  j j k j ƒ  sä t	 ‚ d  S(   NRl   R0   R¥   R  R7  i   i   i   i   R8  R”   g      ð?g       @g      @g      @R¤   R9  (   R\   RB   RA   R:  R   R   R  R   t   listdirR,   R’   R   R   RJ   R7  RH   RD   (   t   df0R1   t   dirRK   (    (    s?   lib/python2.7/site-packages/dask/dataframe/io/tests/test_csv.pyt   test_to_csv_simple  s    +*c       
   C` sÚ   t  j d d d d g d d d d d	 g ƒ}  t j |  d
 d ƒ} t ƒ  n } t | ƒ } | j | d t ƒt j	 | ƒ s‚ t
 ‚ t j t j j | d ƒ d d  d d g ƒj ƒ  } Wd  QX| j |  k j ƒ  sÖ t
 ‚ d  S(   NRl   R0   R¥   R  R”   g      ð?g       @g      @g      @R¤   i   R8   R9  R{   R7  (   R\   R  RA   R:  R   R   R  RC   R   RJ  R,   R’   R   R   R×   RJ   R7  RD   (   RK  R1   RL  RK   (    (    s?   lib/python2.7/site-packages/dask/dataframe/io/tests/test_csv.pyt   test_to_csv_series©  s    -!c       
   ` sñ   d d l  m ‰ t g ‰  ‡  ‡ f d †  }  t j i d d d d g d 6d	 d
 d d g d 6ƒ } t j | d d
 ƒ} t ƒ  m } | j | d t d |  ƒˆ  d sª t	 ‚ t j
 t j j | d ƒ ƒ j ƒ  j d t ƒ } t | | ƒ Wd  QXd  S(   Ni    (   t   getc          ` s   t  ˆ  d <ˆ |  | Ž  S(   Ni    (   R?   (   t   argsRF   (   t   flagt   mp_get(    s?   lib/python2.7/site-packages/dask/dataframe/io/tests/test_csv.pyt   my_get¹  s    
Rl   R0   R¥   R  R7  i   i   i   i   R8  R¤   R”   Rf   R9  RM   (   t   dask.multiprocessingRO  RC   R\   RB   RA   R:  R   R  R,   R’   R   R   R   RJ   RQ   R?   R   (   RS  R1   R  R;  RK   (    (   RQ  RR  s?   lib/python2.7/site-packages/dask/dataframe/io/tests/test_csv.pyt   test_to_csv_with_getµ  s    	0c          C` sp   t  j i t d ƒ d 6ƒ }  t j |  d d ƒ} | j d ƒ d d g k sR t ‚ t j d ƒ t j d ƒ d  S(   Ni
   Rm   R¤   i   s   foo*.csvs   foo0.csvs   foo1.csv(	   R\   RB   R›   RA   R:  R  R,   R   t   remove(   R1   R  (    (    s?   lib/python2.7/site-packages/dask/dataframe/io/tests/test_csv.pyt   test_to_csv_pathsÈ  s
    !s   header, expectedR&   s   x,y
c      
   C` sð   t  j i g  d 6g  d 6ƒ } t j | d d ƒ} t ƒ  ¯ } | j t j j | d ƒ d t	 d |  ƒt j j
 t j j | d ƒ ƒ sŽ t ‚ t j j | d	 ƒ } t | d
 ƒ % } | j ƒ  } | | k sÓ t ‚ Wd  QXt j | ƒ Wd  QXd  S(   NR7  R8  R¤   i   s	   fooe*.csvR”   R8   s	   fooe1.csvs	   fooe0.csvR<  (   R\   RB   RA   R:  R   R  R   R   R   RC   t   existsR,   RÅ   t   readlineRV  (   R8   RP   t   dfet   ddfeR;  RŒ   t   fpt   line(    (    s?   lib/python2.7/site-packages/dask/dataframe/io/tests/test_csv.pyt"   test_to_csv_header_empty_dataframeÐ  s    ((s?   header,header_first_partition_only,expected_first,expected_nexts   a,1
s   d,4
t   aat   bbs   aa,bb
c         C` sP  d } t  j i d d d d d d g d 6d	 d d
 d d d g d 6ƒ } t j | d | ƒ} t ƒ  å } | j t j j | d ƒ d t	 d |  d | ƒt j j | d ƒ } t
 | d ƒ % }	 |	 j ƒ  }
 |
 | k sÛ t ‚ Wd  QXt j | ƒ t j j | d ƒ } t
 | d ƒ % }	 |	 j ƒ  }
 |
 | k s3t ‚ Wd  QXt j | ƒ Wd  QXd  S(   Ni   Rl   R0   R¥   R  Rú   Ry   R7  i   i   i   i   i   R8  R¤   s	   fooa*.csvR”   R8   t   header_first_partition_onlys	   fooa0.csvR<  s	   fooa1.csv(   R\   RB   RA   R:  R   R  R   R   R   RC   RÅ   RY  R,   RV  (   R8   Ra  t   expected_firstt   expected_nextt   partition_countR1   R  R;  RŒ   R\  R]  (    (    s?   lib/python2.7/site-packages/dask/dataframe/io/tests/test_csv.pyt   test_to_csv_headerà  s"    	"'(˜   t
   __future__R    R   R   Rí   R   R   RÄ   R&  R   Ro   Rä   R\   RA   t   toolzR   R   t   pandas.util.testingt   utilt   testingR¾   R;   t   dask.dataframeRì   t	   dask.baseR   t   dask.dataframe.io.csvR	   R
   R   t   dask.dataframe.utilsR   R   t   dask.bytes.coreR   t
   dask.utilsR   R   R   R   t   dask.bytes.compressionR   R   t   cfilesR   RÒ   R×   t   fmt_bsR   R   R   R‘   R!   t   tsv_textt	   tsv_text2RŸ   RÎ   RZ   t	   tsv_filest	   fwf_filesR]   R>   R#   R’   RP   RV   t   csv_units_rowt   tsv_units_rowt   markt   filterwarningst   read_table_markt   parametrizet   paramt
   read_tablet   read_fwft   csv_and_tableR2   R4   R7   R9   RI   RL   RU   Rc   Re   t
   csv_blockst
   tsv_blocksRk   Rr   Rz   R}   R   R‚   R†   R‹   R   RŽ   R   Rš   Rœ   R    R¦   R©   Rª   R®   R¶   RÁ   t   slowRË   RÌ   RÏ   RÑ   RÔ   RÛ   RÜ   RÝ   Rß   Rà   Rõ   R÷   Rû   Rÿ   R   R  R  R  R  R  R  R  R   R#  R$  R(  R,  R.  R/  R2  R4  R6  R=  RF  t   xfailRI  RM  RN  RU  RW  RC   R?   R^  Re  (    (    (    s?   lib/python2.7/site-packages/dask/dataframe/io/tests/test_csv.pyt   <module>   s<  "<		



	2	
		

!!				
					6																		j	"				'	
										'				0