ó ¦–Õ\c@`sddlmZmZmZddlmZddlmZddlZddl m Z ddl Z ddl ZddlmZddlmZdd lmZmZdd lmZdd lmZd d lmZmZmZmZmZd dl m!Z!d dl"m#Z#m$Z$m%Z%ddl"m&Z&m'Z'e ƒZ(ddd„Z*ddd„Z+dde,dd„Z-de,de(d„Z.dde(d„Z/ddd„Z0d„Z1e2d„Z3e2d„Z4d„Z5e#dddd„ƒZ6ddd„Z7ere5j8ej5_8e4j8ej4_8ndS(i(tabsolute_importtdivisiontprint_function(tceil(tgetitemN(tLock(tmergei(ttokenize(tunicodetPY3(tarray(tdelayedi(t DataFrametSeriestIndext new_dd_objectthas_parallel_type(t set_partition(tinsert_meta_param_descriptiont check_metat make_meta(tMt ensure_dictc`sÛˆjdkr(tdˆjfƒ‚n|dk r^t|tƒsRtdƒ‚n|j}ntˆjddƒdk rU|dkršt ˆjj ƒ}nyt j |ƒr¸tdƒ‚n[t ‡fd†|Dƒƒstt|ƒjˆjj ƒƒ}tdjˆj|ƒƒ‚nˆjj}g|D]&}||krF||dnd ^q&}nTˆjd krít j |ƒs|dkr¡tjgd |d ˆjd |ƒSt|ƒd krÞtjt jgd ˆjƒd|d |ƒStdƒ‚n¼t jˆjd ƒrtdƒ‚n|dkrRˆjdkrFt tˆjd ƒƒndg}nAt|ƒˆjd kr“tdjt|ƒˆjd ƒƒ‚nˆjgt|ƒ}d„t||ƒDƒ}tj|d|d |ƒS(s@ Create empty pd.DataFrame or pd.Series which has correct dtype isEfrom_array does not input more than 2D array, got array with shape %rs3'index' must be an instance of dask.dataframe.Indextnamess+For a struct dtype, columns must be a list.c3`s!|]}|ˆjjkVqdS(N(tdtypeR(t.0ti(tx(s3lib/python2.7/site-packages/dask/dataframe/io/io.pys ,ss!dtype {0} doesn't have fields {1}itf8itnameRtindextcolumnss?For a 1d array, columns must be a scalar or single element lists Shape along axis 1 must be knownsSNumber of column names must match width of the array. Got {0} names for {1} columnscS`s.i|]$\}}tjgd|ƒ|“qS(R(tnpR (Rtctdt((s3lib/python2.7/site-packages/dask/dataframe/io/io.pys Ds N(tndimt ValueErrortshapetNonet isinstanceRt_metatgetattrRtlistRR tisscalartalltsortedtsett differencetformattfieldstpdR tlenR R tisnantrangetzip(RRRtextraR1tntdtypestdata((Rs3lib/python2.7/site-packages/dask/dataframe/io/io.pyt_meta_from_arraysD   !  6"+ 4 iPÃc C`sNt|tjƒr"t|d|ƒSt||ƒ}ttdt|ƒ|ƒƒ}|t|ƒdf}t|||ƒ}d|}i}x°tdt t t|ƒ|ƒƒƒD]‰}t |t |||d|ƒf} t|t jƒrt j| d|j|jf|||f>> df = pd.DataFrame(dict(a=list('aabbcc'), b=list(range(6))), ... index=pd.date_range(start='20100101', periods=6)) >>> ddf = from_pandas(df, npartitions=3) >>> ddf.divisions # doctest: +NORMALIZE_WHITESPACE (Timestamp('2010-01-01 00:00:00', freq='D'), Timestamp('2010-01-03 00:00:00', freq='D'), Timestamp('2010-01-05 00:00:00', freq='D'), Timestamp('2010-01-06 00:00:00', freq='D')) >>> ddf = from_pandas(df.a, npartitions=3) # Works with Series too! >>> ddf.divisions # doctest: +NORMALIZE_WHITESPACE (Timestamp('2010-01-01 00:00:00', freq='D'), Timestamp('2010-01-03 00:00:00', freq='D'), Timestamp('2010-01-05 00:00:00', freq='D'), Timestamp('2010-01-06 00:00:00', freq='D')) Raises ------ TypeError If something other than a ``pandas.DataFrame`` or ``pandas.Series`` is passed in. See Also -------- from_array : Construct a dask.DataFrame from an array that has record dtype read_csv : Construct a dask.DataFrame from a CSV file Rs,Dask does not support MultiIndex Dataframes.s*Input must be a pandas DataFrame or Seriess;Exactly one of npartitions and chunksize must be specified.s from_pandas-it ascendingRBc`s5i|]+\}\}}ˆj||!ˆ|f“qS((tiloc(RRtstarttstop(R:R(s3lib/python2.7/site-packages/dask/dataframe/io/io.pys Ès iÿÿÿÿiN(R'R)R&R2t MultiIndextNotImplementedErrorRt TypeErrorR$R3R@RRRRtis_monotonic_increasingt sort_indextTruetsorted_division_locationsR*R5t enumerateR6( R:t npartitionsRBtsortRtnrowsRDt locationsRF((R:Rs3lib/python2.7/site-packages/dask/dataframe/io/io.pyt from_pandasms.=   & %!c `s6ˆtkrtƒ‰nddlj}ddl}tˆttfƒr]|jdˆƒ‰nt ‡fd†ˆj Dƒƒ}ˆdkr|dkr|‰nt ƒ‰|r\x­ˆj D]Ÿ} t jˆj| t jƒst jˆj| t jƒst jˆj| t jƒr¶|jˆ| dˆtˆj ƒfƒ} |j| ƒjƒˆ| ìsi'tchunksis from_bcolz-c3`sK|]A}ˆ|ftˆt|ˆ|dˆƒfˆˆˆffVqdS(iN(tdataframe_from_ctableRA(RR(t categoriesRBRtlocktnew_nameR(s3lib/python2.7/site-packages/dask/dataframe/io/io.pys sidi()RQRt dask.arrayR tbcolzR'tstrRtctabletmaxRR&tdictR t issubdtypeRtstring_tunicode_tobject_RGR3tuniquetcomputeR?R5RYRtostpathtgetmtimetidR%R@RR\RAR tAssertionErrortlinspacet percentileR(RRBt categorizeRR^tkwargsR<Rat bc_chunklenRtaRDRERFRCtresulttq((R]RBRR^R_Rs3lib/python2.7/site-packages/dask/dataframe/io/io.pyt from_bcolzÍsJ     )# !!  ,!)#c C`svddl}|dkr'|jj}nt|tƒrEt|ƒ}n||}t|ƒtkr—|j }|j t |ƒkrˆ|j n t |ƒ}n?|dj }|dj t |ƒkrÊ|dj n t |ƒ}t j t||ƒƒ}|r|jƒnzZt||jƒrÓg|D]} || |^q} |dk r¦gt|| ƒD]J\} } | |kr—t jjtj|| | ƒ|| tƒn| ^qS} nt jtt|| ƒƒd|d|ƒ} n‡t||jƒrZ||} |dk r<|r<||kr<t jjtj||| ƒ||tƒ} nt j| d|d|ƒ} nWd|rq|jƒnX| S(s Get DataFrame from bcolz.ctable Parameters ---------- x: bcolz.ctable slc: slice columns: list of column names or None >>> import bcolz >>> x = bcolz.ctable([[1, 2, 3, 4], [10, 20, 30, 40]], names=['a', 'b']) >>> dataframe_from_ctable(x, slice(1, 3)) a b 1 2 20 2 3 30 >>> dataframe_from_ctable(x, slice(1, 3), columns=['b']) b 1 20 2 30 >>> dataframe_from_ctable(x, slice(1, 3), columns='b') 1 20 2 30 Name: b, dtype: int... iNRRR(RaR&RRR'R?R*ttypeRARJRKR3R2RR5tacquireRcR6t Categoricalt from_codesR t searchsortedRQR RetcarrayR trelease( RtslcRR]R^RaRJRKtidxRR[tchunkRw((s3lib/python2.7/site-packages/dask/dataframe/io/io.pyR\s@    - 2 ! `!   "cC`sÏt|||ƒ}|jdkrZt|jdƒdkrZ|ji|jdd6ƒ}ndt||ƒ}g}|d k r t|t ƒst dƒ‚n|j |j dkrÞdj |j |j dƒ}t |ƒ‚n|j}|jt|jƒƒ|jƒ}nÛtjt|jƒƒr_d gt|jdƒd}d gt|jdƒ}nˆdg}x*|jdD]}|j|d|ƒqvWgt|d |dƒD]$\} } tj| | ddf^q­}|dcd8>> import dask.array as da >>> import dask.dataframe as dd >>> x = da.ones((4, 2), chunks=(2, 2)) >>> df = dd.io.from_dask_array(x, columns=['a', 'b']) >>> df.compute() a b 0 1.0 1.0 1 1.0 1.0 2 1.0 1.0 3 1.0 1.0 See Also -------- dask.bag.to_dataframe: from dask.bag dask.dataframe._Frame.values: Reverse conversion dask.dataframe._Frame.to_records: Reverse conversion iisfrom-dask-arrays3'index' must be an instance of dask.dataframe.Indexis>'index' must have the same number of blocks as 'x'. ({} != {})iÿÿÿÿti8N("R;R#R3R[trechunkR%RR&R'RR$RTt numblocksR0RDtappendRtdaskt __dask_keys__R R4tsumR6tarangeRSR2R RRR RtextendRR(RRRRCRtto_mergetmsgRDR!RvtbRFRRƒtind((s3lib/python2.7/site-packages/dask/dataframe/io/io.pyR>]sB)(    ?. +&cC`sdS(s¹ A dummy function to link results together in a graph We use this to enforce an artificial sequential ordering on tasks that don't explicitly pass around a shared resource N(R&(RERw((s3lib/python2.7/site-packages/dask/dataframe/io/io.pyt_link³scC`sdt|tjƒr.ttt|j|ƒƒƒSt|tjƒr`|rVt|jƒƒSt|ƒSdS(N( R'R2R R*tmapR?t itertuplesR t iteritems(tdfR((s3lib/python2.7/site-packages/dask/dataframe/io/io.pyt _df_to_bag¼sc`sªddlm}t|ttfƒs4tdƒ‚ndt|ˆƒ‰t‡‡fd†t|j ƒƒDƒƒ}|j |j |j ƒ|j ƒƒƒ||ˆ|j ƒS(s,Create Dask Bag from a Dask DataFrame Parameters ---------- index : bool, optional If True, the elements are tuples of ``(index, value)``, otherwise they're just the ``value``. Default is False. Examples -------- >>> bag = df.to_bag() # doctest: +SKIP i(tBags%df must be either DataFrame or Seriessto_bag-c3`s0|]&\}}ˆ|ft|ˆffVqdS(N(R–(RRtblock(RR(s3lib/python2.7/site-packages/dask/dataframe/io/io.pys Ôs(tbag.coreR—R'R R RNRReRSR‰tupdatet__dask_optimize__t__dask_graph__RT(R•RR—RF((RRs3lib/python2.7/site-packages/dask/dataframe/io/io.pytto_bagÃs %cC`s|jtjƒS(sì Create Dask Array from a Dask Dataframe Warning: This creates a dask.array without precise shape information. Operations that depend on shape information, like slicing or reshaping, will not work. Examples -------- >>> df.to_records() # doctest: +SKIP dask.array See Also -------- dask.dataframe._Frame.values dask.dataframe.from_dask_array (tmap_partitionsRt to_records(R•((s3lib/python2.7/site-packages/dask/dataframe/io/io.pyRŸÚss from-delayedc `sÜddlm}t||ƒr+|g}ng|D]7}t||ƒ rct|dƒrct|ƒn|^q2}x9|D]1}t||ƒsvtdt|ƒjƒ‚qvqvWˆd krÊ|dj ƒ‰nt ˆƒ‰|dt |Œ‰t d„|Dƒƒ}|j ‡‡fd†t|ƒDƒƒ|d ks>|dkrXd gt|ƒd }n7t|ƒ}t|ƒt|ƒd krtd ƒ‚nt|ˆˆ|ƒ}|dkrØd d lm}||ƒ}||_n|S(sÖ Create Dask DataFrame from many Dask Delayed objects Parameters ---------- dfs : list of Delayed An iterable of ``dask.delayed.Delayed`` objects, such as come from ``dask.delayed`` These comprise the individual partitions of the resulting dataframe. $META divisions : tuple, str, optional Partition boundaries along the index. For tuple, see https://docs.dask.org/en/latest/dataframe-design.html#partitions For string 'sorted' will compute the delayed values to find index values. Assumes that the indexes are mutually sorted. If None, then won't use index information prefix : str, optional Prefix to prepend to the keys. i(tDelayedtkeysExpected Delayed object, got %st-cs`s|]}|jVqdS(N(Rˆ(RR•((s3lib/python2.7/site-packages/dask/dataframe/io/io.pys sc`s4i|]*\}}t|jˆdfˆ|f“qS(t from_delayed(RR¡(RRR•(RCR(s3lib/python2.7/site-packages/dask/dataframe/io/io.pys s R-is+divisions should be a tuple of len(dfs) + 1i(tcompute_divisionsN(t dask.delayedR R'thasattrR RNRzt__name__R&RkRRRRšRSR3R?R$RtshuffleR¤RD( tdfsRCRDtprefixR R•RFtdivsR¤((RCRs3lib/python2.7/site-packages/dask/dataframe/io/io.pyR£îs6 A       cC`sb|dk|dkkr'tdƒ‚n|rFtt|ƒ|ƒ}ndg}|dg}x¼ttdt|ƒ|ƒƒD]œ}||dkr—q{nx<|dt|ƒkrÕ||d||krÕ|d7}qšW|j||ƒ|t|ƒdkr |d7}n|j|ƒq{W|dt|ƒkrX|jt|ƒƒ|j|dƒn||fS(sZ Find division locations and values in sorted list Examples -------- >>> L = ['A', 'B', 'C', 'D', 'E', 'F'] >>> sorted_division_locations(L, chunksize=2) (['A', 'C', 'E', 'F'], [0, 2, 4, 6]) >>> sorted_division_locations(L, chunksize=3) (['A', 'D', 'F'], [0, 3, 6]) >>> L = ['A', 'A', 'A', 'A', 'B', 'B', 'B', 'C'] >>> sorted_division_locations(L, chunksize=3) (['A', 'B', 'C'], [0, 4, 8]) >>> sorted_division_locations(L, chunksize=2) (['A', 'B', 'C'], [0, 4, 8]) >>> sorted_division_locations(['A'], chunksize=2) (['A', 'A'], [0, 1]) s;Exactly one of npartitions and chunksize must be specified.iiÿÿÿÿiN(R&R$RR3R*R5R‡(tseqRTRBt positionstvaluestpos((s3lib/python2.7/site-packages/dask/dataframe/io/io.pyRR(s&  %1 (9t __future__RRRtmathRtoperatorRRlt threadingRtpandasR2tnumpyR ttoolzRtbaseRt compatibilityRR tR R<R tcoreR R RRRR¨RtutilsRRRRRR^R&R;RGRQRXRyR\R>R‘tFalseR–RRŸR£RRt__doc__(((s3lib/python2.7/site-packages/dask/dataframe/io/io.pyts@   ( .%`KEV   90