
\c           @` s;  d  d l  m Z m Z m Z d  d l Z d  d l Z d  d l Z d  d l m Z d  d l	 m
 Z
 d  d l Z d  d l Z e j d  Z e j d  Z e j d  Z e j d  Z d  d	 l m Z m Z m Z d  d
 l m Z d  d l m Z d  d l m Z m Z m Z d  d l m  Z  m! Z" m# Z# e e d d Z d Z$ i d d 6d d 6Z! e d    Z% e j&   d    Z' e d    Z( e j&   e j) j* d     Z+ d   Z, d   Z- d   Z. d   Z/ d   Z0 d   Z1 d   Z2 e j) j3 d  d! d" d# d$ g  d%    Z4 e j) j3 d  d! d" d# d$ g  d&    Z5 e j) j3 d' g  e" D] Z6 e6 d f ^ qNg  e# D] Z6 e6 d( f ^ qj d)    Z8 e j) j3 d* d+ d, g  d-    Z9 d.   Z: d/   Z; d0   Z< d1   Z= e j) j3 d2 d3 d4 g  d5    Z> d6   Z? e j) j@ e jA d7 k d8 d9 d:    ZB d;   ZC d S(<   i    (   t   print_functiont   divisiont   absolute_importN(   t   contextmanager(   t   LooseVersiont   s3fst   boto3t   motot	   httpretty(   t   concatt   valmapt   partial(   t   compute(   t   DaskS3FileSystem(   t
   read_bytest
   open_filest   get_pyarrow_filesystem(   t   compresst   filest   seekable_filest	   schedulert   synct   tests   {"amount": 100, "name": "Alice"}
{"amount": 200, "name": "Bob"}
{"amount": 300, "name": "Charlie"}
{"amount": 400, "name": "Dennis"}
s   test/accounts.1.jsons   {"amount": 500, "name": "Alice"}
{"amount": 600, "name": "Bob"}
{"amount": 700, "name": "Charlie"}
{"amount": 800, "name": "Dennis"}
s   test/accounts.2.jsonc          c` s=   t  t j  }  z	 d VWd t j j   t j j |   Xd S(   s   
    Get a context manager to safely set environment variables
    All changes will be undone on close, hence environment variables set
    within this contextmanager will neither persist nor change global state.
    N(   t   dictt   ost   environt   cleart   update(   t   saved_environ(    (    s7   lib/python2.7/site-packages/dask/bytes/tests/test_s3.pyt!   ensure_safe_environment_variables'   s
    	c          c` s   t     t j j d d  t j j d d  d d  l }  |  j    t j d  } | j d t	 d d	  x6 t
 j   D]( \ } } | j d t	 d
 | d |  q{ Wt j d t  Vt j j   t j j   Wd  QXWd  QXd  S(   Nt   AWS_ACCESS_KEY_IDt
   foobar_keyt   AWS_SECRET_ACCESS_KEYt   foobar_secreti    t   s3t   Buckett   ACLs   public-read-writet   Keyt   Bodyt   anon(   R   R   R   t
   setdefaultR   t   mock_s3R   t   clientt   create_buckett   test_bucket_nameR   t   itemst
   put_objectR   t   S3FileSystemt   TrueR   t	   HTTPrettyt   disablet   reset(   R   R*   t   ft   data(    (    s7   lib/python2.7/site-packages/dask/bytes/tests/test_s3.pyR"   6   s    
 c         c` s,  t    t j j d d  t j j d d  t j    t j d  } | j d |  d d  x6 | j	   D]( \ } } | j
 d |  d	 | d
 |  qo Wt d t  Vxp | j	   D]b \ } } z8 y  | j d |  d	 | d
 |  Wn t k
 r n XWd  t j j   t j j   Xq WWd  QXWd  QXd  S(   NR   R   R    R!   R"   R#   R$   s   public-read-writeR%   R&   R'   (   R   R   R   R(   R   R)   R   R*   R+   R-   R.   R   R0   t   delete_objectt	   ExceptionR   R1   R2   R3   (   t   bucketR   R*   R4   R5   (    (    s7   lib/python2.7/site-packages/dask/bytes/tests/test_s3.pyt
   s3_contextK   s"    
   c   
      c` s  t  j d  } i i d d 6d d 6d d 6d d 6d d 6d 6i d d 6d	 d 6d	 d 6d
 d 6d
 d 6d 6i d d 6d d 6d d 6d d 6d d 6d 6i d d 6d d 6d d 6d d 6d d 6d 6i d d 6d d 6d d 6d d 6d d 6d 6i d d 6d d 6d d 6d d 6d d 6d 6i d d 6d  d 6d! d 6d" d 6d# d 6d$ 6i d d 6d d 6d d 6d d 6d d 6d% 6i d& d 6d& d 6d& d 6d& d 6d& d 6d' 6i d( d 6d) d 6d* d 6d+ d 6d, d 6d- 6i d. d 6d/ d 6d0 d 6d1 d 6d2 d 6d3 6i d d 6d d 6d d 6d d 6d d 6d4 6i d5 d 6d6 d 6d7 d 6d8 d 6d9 d 6d: 6i d; d 6d d 6d d 6d d 6d d 6d< 6i d d 6d d 6d d 6d d 6d d 6d= 6i d> d 6d? d 6d@ d 6d@ d 6d@ d 6dA 6i d@ d 6d@ d 6d@ d 6d@ d 6d@ d 6dB 6i dC d 6dC d 6dC d 6dC d 6dC d 6dD 6i dE d 6dF d 6dG d 6dH d 6dI d 6dJ 6} | j |  } | j t j dK  j dL   } t j   } t j	 |  } | j
 | dM t dN } t j dO  } | j dP t dQ | dR |  dS } xI t d dT  D]8 }	 | j d  | j dP t dQ | j |	  dR |  qWdU VdU S(V   s   
    Fixture with sample yellowtrip CSVs loaded into S3.

    Provides the following CSVs:

    * s3://test/nyc-taxi/2015/yellow_tripdata_2015-01.csv
    * s3://test/nyc-taxi/2014/yellow_tripdata_2015-mm.csv
      for mm from 01 - 12.
    t   pandasi   i    i   i   i   t   VendorIDs   2015-01-15 19:05:39s   2015-01-10 20:33:38s   2015-01-10 20:33:39t   tpep_pickup_datetimes   2015-01-15 19:23:42s   2015-01-10 20:53:28s   2015-01-10 20:43:41s   2015-01-10 20:35:31s   2015-01-10 20:52:58t   tpep_dropoff_datetimet   passenger_countgq=
ףp?gffffff
@g?g      ?g      @t   trip_distanceg    Rg    Rg   `}Rg   Rg   '~Rt   pickup_longitudeg   `D@g    \D@g   fD@g   `^[D@g   @aD@t   pickup_latitudet
   RateCodeIDt   Nt   store_and_fwd_flagg   b~Rg   Rg   |Rg  FRg   DRt   dropoff_longitudeg   @`D@g   *aD@g   `iD@g   (\D@g   @_D@t   dropoff_latitudet   payment_typeg      (@g      -@g      #@g      @g      .@t   fare_amountg      ?t   extrat   mta_taxg      
@g       @g        t
   tip_amountt   tolls_amountg333333?t   improvement_surchargeg1@g1@g%@g333333@gL0@t   total_amounti   i'  t   indexs)   nyc-taxi/2015/yellow_tripdata_2015-01.csvR"   R#   R%   R&   s.   nyc-taxi/2014/yellow_tripdata_2014-{:0>2d}.csvi   N(   t   pytestt   importorskipt	   DataFramet   taket   npt   aranget   repeatt   iot   BytesIOt   TextIOWrappert   to_csvt   FalseR   R*   R.   R,   t   ranget   seekt   format(
   R"   t   pdR5   t   samplet   dft   filet   sfilet   keyR*   t   i(    (    s7   lib/python2.7/site-packages/dask/bytes/tests/test_s3.pyt   s3_with_yellow_tripdatae   sv    -

**

**

*******-!c       
   C` s   t  d d d d  }  |  j d k s* t  |  j d k s? t  t  d d d d  }  |  j d k si t  |  j d k s~ t  t j t   t  d d d d  Wd  QXt j t   t  d d d d  Wd  QXd  S(   NRd   t   secrett   usernamet   password(   R   Rd   t   AssertionErrorRg   RP   t   raisest   KeyError(   R"   (    (    s7   lib/python2.7/site-packages/dask/bytes/tests/test_s3.pyt   test_get_s3   s    c   	      C` s   g  t  D] } d t d | ^ q } t | d d } x? t | t  j    D]( \ } } |  } | j |  Wd  QXqM Wt d t d  \ } } t t |    } t	 t
 t  j     t	 |  k s t  d  S(   Ns   s3://s   /more/t   modet   wbs   /more/test/accounts.*(   R   R,   R   t   zipt   valuest   writeR   R   R	   t   sett   listRj   (	   R"   R4   t   pathst   filst   filR5   R`   Rq   t   results(    (    s7   lib/python2.7/site-packages/dask/bytes/tests/test_s3.pyt   test_open_files_write   s    %"	c         C` s$  t  d t d  \ } } t | t  s/ t  | d  t t t  d d  k sW t  | j d  sl t  t | t t	 f  s t  t | d t t	 f  s t  t
 | d d d  s t  t t t |   t t  k s t  t t |    } t |  t t j    k s t  d  S(   Ns   s3://s   /test/accounts.*i   i    s   
t   dask(   R   R,   t
   isinstancet   bytesRj   R   t   sortedt   endswithRt   t   tuplet   hasattrt   sumt   mapt   lenR   R	   Rs   Rq   (   R"   R`   Rq   Rx   (    (    s7   lib/python2.7/site-packages/dask/bytes/tests/test_s3.pyt   test_read_bytes   s    ('c         C` s   t  d t d d d d d \ } } | j d  s; t  t  d t d d d d d \ } } | j d  sv t  t  d t d d d d d \ } } | j d  s t  d  S(	   Ns   s3://s   /test/accounts.*R`   iP   t	   delimiters   
s   /test/accounts.1.jsoni   (   R   R,   R~   Rj   (   R"   R`   Rq   (    (    s7   lib/python2.7/site-packages/dask/bytes/tests/test_s3.pyt    test_read_bytes_sample_delimiter   s    c         C` s,   t  j t   t d t d  Wd  QXd  S(   Ns   s3://s   /non-existing/*(   RP   Rk   t   IOErrorR   R,   (   R"   (    (    s7   lib/python2.7/site-packages/dask/bytes/tests/test_s3.pyt!   test_read_bytes_non_existing_glob   s    c         C` sK   t  d t d d d  \ } } t t t |   t t  k sG t  d  S(   Ns   s3://s   /test/accounts.*t	   blocksize(   R   R,   t   NoneR   R   R   R   Rj   (   R"   t   _Rq   (    (    s7   lib/python2.7/site-packages/dask/bytes/tests/test_s3.pyt   test_read_bytes_blocksize_none   s    c         C` s   t  d j t  d d  d t \ } } t |  d k s? t  t  d j t  d d  d t \ } } t |  d k s~ t  d  S(   Ns1   s3://{}/nyc-taxi/2015/yellow_tripdata_2015-01.csvR   R'   i   s   s3://{}/nyc-taxi/2014/*.csvi   (   R   R^   R,   R   R0   R   Rj   (   Rf   R   t   L(    (    s7   lib/python2.7/site-packages/dask/bytes/tests/test_s3.pyt'   test_read_bytes_blocksize_on_large_data   s    	R   i   i   i-   i  c         C` s
  t  d t d d | \ } } t t t |   g  t j   D] } t |  | d ^ q? k sh t  t t	 |    } t
 d   | D  t
 d   t j   D  k s t  d j |  j d  } d j t j    j d  } t |  t |  k st  d  S(	   Ns   s3://s   /test/account*R   i   c         s` s   |  ] } t  |  Vq d  S(   N(   R   (   t   .0t   r(    (    s7   lib/python2.7/site-packages/dask/bytes/tests/test_s3.pys	   <genexpr>  s    c         s` s   |  ] } t  |  Vq d  S(   N(   R   (   R   t   v(    (    s7   lib/python2.7/site-packages/dask/bytes/tests/test_s3.pys	   <genexpr>  s    t    s   
(   R   R,   Rt   R   R   R   Rq   Rj   R   R	   R   t   joint   splitRs   (   R"   R   R   t   valsR   Rx   t   ourlinest	   testlines(    (    s7   lib/python2.7/site-packages/dask/bytes/tests/test_s3.pyt   test_read_bytes_block   s    6%c         C` s  t  d t d d | d d \ } } t  d t d d | d d \ } } g  t |  D] } | j ^ qY g  t |  D] } | j ^ qx k s t  t t |    } g  | D] } | r | ^ q }	 t d   |	 D  s t  d j |	  j d  }
 d j d	   t	 t
  D  j d  } |
 | k s5t  d
 } t  d t d d | d | \ } } t t |    } g  | D] } | rz| ^ qz}	 t d   |	 D  t |	  d k st  d j |	  } d j d   t	 t
  D  } | | k st  d  S(   Ns   s3://s   /test/accounts*R   R   s   
t   fooc         s` s   |  ] } | j  d   Vq d S(   s   
N(   R~   (   R   R   (    (    s7   lib/python2.7/site-packages/dask/bytes/tests/test_s3.pys	   <genexpr>  s    R   c         s` s   |  ] } t  | Vq d  S(   N(   R   (   R   t   k(    (    s7   lib/python2.7/site-packages/dask/bytes/tests/test_s3.pys	   <genexpr>  s    t   }c         s` s   |  ] } | j  d   Vq d S(   R   N(   R~   (   R   R   (    (    s7   lib/python2.7/site-packages/dask/bytes/tests/test_s3.pys	   <genexpr>$  s    i   c         s` s   |  ] } t  | Vq d  S(   N(   R   (   R   R   (    (    s7   lib/python2.7/site-packages/dask/bytes/tests/test_s3.pys	   <genexpr>&  s    (   R   R,   R	   Rd   Rj   R   t   allR   R   R}   R   R   R   (   R"   R   R   Rq   t   values2t   at   bRx   R   t   resR   R   t   dt   oursR   (    (    s7   lib/python2.7/site-packages/dask/bytes/tests/test_s3.pyt   test_read_bytes_delimited  s*    +(,s   fmt,blocksizei
   c      
   C` s   t  d t t | t    t d d | d | \ } } | j t t t  d d   sb t  | j d  sw t  t	 t
 |    } d j |  d j g  t t  D] } t | ^ q  k s t  Wd  QXd  S(	   NR   s   s3://compress/test/accounts.*t   compressionR   i    i
   s   
R   (   R9   R
   R   R   R   t
   startswithR}   Rj   R~   R   R	   R   (   R"   t   fmtR   R`   Rq   Rx   R   (    (    s7   lib/python2.7/site-packages/dask/bytes/tests/test_s3.pyt   test_compression*  s    	'Rn   t   rtt   rbc         C` s   t  d t d d | } t |  t t  k s8 t  xr t | t t   D][ \ } } | G } | j   } t | } | d k r | | k n	 | j   s t  Wd  QXqN Wd  S(   Ns   s3://s   /test/accounts.*Rn   R   (	   R   R,   R   R   Rj   Rp   R}   t   readt   decode(   R"   Rn   t   myfilest	   lazy_filet   pathR4   R5   t   sol(    (    s7   lib/python2.7/site-packages/dask/bytes/tests/test_s3.pyt   test_open_files8  s    	"	
c         C` s   |  d S(   Ni   (    (   t   x(    (    s7   lib/python2.7/site-packages/dask/bytes/tests/test_s3.pyt   <lambda>D  R   c          C` s  t  d t  s t d  \ }  } t d  \ }  } g  t |  D] } | j ^ qA g  t |  D] } | j ^ q` k s~ t  Wd  QXt  d t t t    t d  \ }  } Wd  QXg  t |  D] } | j ^ q g  t |  D] } | j ^ q k s t  d  S(   NR   s   s3://compress/test/accounts.*(   R9   R   R   R	   t   _keyRj   R
   t   double(   R   R   R   t   aat   bbt   ct   cc(    (    s7   lib/python2.7/site-packages/dask/bytes/tests/test_s3.pyt!   test_modification_time_read_bytesG  s    Pc       
   C` so   t  j d  }  t d i d d 6 D } |  j d d i | d 6} | j j   j   d
 k se t  Wd  QXd  S(   Ns   dask.dataframet   csvs   a,b
1,2
3,4s   a.csvs   s3://csv/*.csvt   storage_optionsR"   i   i   i   (   RP   RQ   R9   t   read_csvR   R   R   Rj   (   t   ddR"   Ra   (    (    s7   lib/python2.7/site-packages/dask/bytes/tests/test_s3.pyt$   test_read_csv_passes_through_optionsT  s    c       
   C` sr   t  j d  }  t d i d d 6 G } |  j d d i | d 6} | j   j d d	  d
 k sh t  Wd  QXd  S(   Ns   dask.bagR   s   a,b
1,2
3,4s   a.csvs   s3://csv/*.csvR   R"   R   R   i   (   RP   RQ   R9   t	   read_textt   countR   Rj   (   t   dbR"   Ra   (    (    s7   lib/python2.7/site-packages/dask/bytes/tests/test_s3.pyt%   test_read_text_passes_through_options[  s    t   enginet   pyarrowt   fastparquetc         C` s  t  j d  } t  j |  } | d k rO t | j  d k rO t  j d  n  d d  l } d d  l } d t } | j i | j	 d d | j
 d	 6| j	 d d | j d
 6| j	 d d | j d 6| j j d d d g d d j d  d 6d | j | j	 d  d d } | j | d d } | j | d | g  |  j |  D] }	 |	 j d  d ^ qL}
 d |
 k s}t  d |
 k st  | j | d d d | } t | j  d k st  | j j j | | j    d  S(   Ns   dask.dataframeR   s   0.13.0s(   pyarrow 0.13.0 not supported for parqueti    s   s3://%s/test.parqueti  t   dtypet   i32t   i64R4   u   hellou   youu   peoplet   sizet   Ot   bhelloRO   t   nameR   t	   chunksizei  R   t   /it   _common_metadatas   part.0.parqueti   (   RP   RQ   R   t   __version__t   skipR:   t   numpyR,   RR   RU   t   int32t   int64t   float64t   randomt   choicet   astypet   Indext   from_pandast
   to_parquett   lsR   Rj   t   read_parquetR   t	   divisionst   utilt   testingt   assert_frame_equalR   (   R"   R   R   t   libR_   RT   t   urlR5   Ra   R4   R   t   df2(    (    s7   lib/python2.7/site-packages/dask/bytes/tests/test_s3.pyt   test_parquetb  s,    !
"	!/c         C` sk  t  j d  } t  j d  d d  l } d d  l } d t } | j i | j d d d d g  d 6 } | j | d d	 } | j | d
 t	 | j
 | d i t	 d 6|  j   j t	 k s t  | j
 | d i t d 6|  j   j t k s t  | j
 | d i d d 6|  j   j d k s0t  |  j   j | d   } | j d k sat  Wd  QXd  S(   Ns   dask.dataframeR   i    s   s3://%s/test.parqueti   i   R   R   i  t   write_indexR   t   default_fill_cachei   t   default_block_sizes
   /_metadatai   i   i   (   RP   RQ   R:   R   R,   RR   t   arrayR   R   R[   R   t   currentR   Rj   R0   R   t   openR   (   R"   R   R_   RT   R   R5   Ra   R4   (    (    s7   lib/python2.7/site-packages/dask/bytes/tests/test_s3.pyt   test_parquet_wstoragepars  s     
+t   win32t   reasons!   pathlib and moto clash on windowsc         C` sP   t  j d  } t  j t  , | j d  } t | d d  \ } } Wd  QXd  S(   Nt   pathlibs   s3://bucket/test.accounts.*R   (   RP   RQ   Rk   t
   ValueErrort   PathR   R   (   R"   R   R   R`   Rq   (    (    s7   lib/python2.7/site-packages/dask/bytes/tests/test_s3.pyt   test_pathlib_s3  s    c         C` sC   t  j d  } t d t  } t t |  | j j  s? t  d  S(   NR   R'   (	   RP   RQ   R   R0   R{   R   t
   filesystemt   S3FSWrapperRj   (   R"   t   pat   fs(    (    s7   lib/python2.7/site-packages/dask/bytes/tests/test_s3.pyt   test_get_pyarrow_fs_s3  s    (D   t
   __future__R    R   R   RW   t   sysR   t
   contextlibR   t   distutils.versionR   RP   R   RT   RQ   R   R   R   R   t   toolzR	   R
   R   Rz   R   t   dask.bytes.s3R   t   dask.bytes.coreR   R   R   t   dask.bytes.compressionR   R   t   compress_filesR   R,   R   t   fixtureR"   R9   t   markt   slowRf   Rm   Ry   R   R   R   R   R   t   parametrizeR   R   R   R   R   R   R   R   R   R   R   R   t   skipift   platformR   R   (    (    (    s7   lib/python2.7/site-packages/dask/bytes/tests/test_s3.pyt   <module>   s^   

	K							**,$				$	