\c@`s<ddlmZmZmZddlZddlZddlmZmZddl m Z m Z ddlm Z ddl mZddlZddlZddlmZmZddlmZmZmZmZmZydd lmZWnek re ZnXd d l!m"Z#d d l!m$Z$d d l%m&Z&m'Z'm(Z(d dl!m)Z)d dl*m+Z+m,Z,m-Z-m.Z.m/Z/m0Z0d dl1m2Z2d dl%m3Z3m4Z4m5Z5m6Z6m7Z7m8Z8m9Z9m:Z:m;Z;m<Z<m=Z=m>Z>d dl?m@Z@mAZAd dlBmBZBmCZCd dlDmEZEmFZFmGZGmHZHd dlImIZImJZJmKZKd dlLmMZMddl!mNZNddlOmPZPmQZQddlRmSZSmTZTddlUmUZUddl%mVZVmWZWmXZXmYZYmZZZm[Z[m\Z\m]Z]m^Z^m_Z_m`Z`maZambZbdZcejddeedZfd Zgd!eEe;fd"YZheed#Zid$eEe;fd%YZjd&Zkd'ejfd(YZld)elfd*YZmd+ejfd,YZnxejoejpejqe,ejrejsejtejuejvejwejxejyejzej{ej|ej}ej~ejejejgD] Zejjeehjeq+Wxd-d.d/d0d1d2d3d4d5d6d7d8d9d:d;d<gD]JZeejneZenjeeeejleZeljeeqWxdd=d>d?d@dAdBgD]JZeejneZenjeeeejleZeljeeqWdCZdDZdEZdFZeedGZdHZdIZedJZeXeeeeceeeeeeeedK ZeZeedLZdMZeXdNZdOZdPZdQZdRdSZeeeeeeedTZeedUZeedVZd eeeedWZedXZedYZdZZeed[Zed\Zd]Zeeed^Zed_Zed`ZedaZeedbZeedcZeeddZeeeedeZdfZdgZeejdhZeejdidjdkZeedlrueejdmZndnZe'doZejejldpZejejndqZejejmdrZejedsZejejdtZduZdvZdwZdxZdyZdS(zi(tabsolute_importtdivisiontprint_functionN(twrapstpartial(tNumbertIntegral(tgetitem(tpformat(tcache_readonlythash_pandas_object(tmergetfirsttuniquet partition_alltremove(tChesti(tarray(tcore(tpartial_by_ordertDispatcht IndexCallable(tthreaded(tapplyt operator_divt bind_methodt string_typestIteratortSequence(t globalmethod( trandom_state_datat pseudorandomt derived_fromtfuncnamet memory_reprt put_linestMt key_splittOperatorMethodMixint is_arraylikettypenamet skip_doctest(tArrayt normalize_arg(t blockwiset Blockwise(tDaskMethodsMixinttokenizet dont_optimizetis_dask_collection(tdelayedtDelayedtunpack_collections(tHighLevelGraphi(tmethods(tDatetimeAccessortStringAccessor(tCategoricalAccessort categorize(toptimize( t meta_nonemptyt make_metatinsert_meta_param_descriptiontraise_on_meta_errortclear_known_categoriestis_categorical_dtypethas_known_categoriestPANDAS_VERSIONt index_summarytis_dataframe_liketis_series_liket is_index_liketvalid_divisionst__no_default__scompute.use_numexprcC`s|s |Stttj|tjr;tjj|St|dstyt j |SWqtt k rp|SXng|D]}t |r{|^q{}|s|dSt j|dtS(Nituniform(t isinstanceR Rtflattentnptndarraytdat concatenate3thas_parallel_typetpdtSeriest ExceptiontlenR6tconcattTrue(targstitargs2((s2lib/python2.7/site-packages/dask/dataframe/core.pyt_concat5s! %cC`s t|S(N(R[(tresults((s2lib/python2.7/site-packages/dask/dataframe/core.pytfinalizeGstScalarcB`seZdZddZdZdZdZdZe e ddde Z e ejZd Zd Zed Zed Zd ZedZdZdZedZdZdZedZedZee dZ!e"dZ#RS(s+ A Dask object to represent a pandas scalarcC`st|ts*tj||dg}n||_||_t|}t|slt|slt|rt dj t t |n||_ dS(Nt dependenciess(Expected meta to specify scalar, got {0}(RKR5tfrom_collectionstdaskt_nameR=RERFRGt TypeErrortformatR(ttypet_meta(tselftdsktnametmetat divisions((s2lib/python2.7/site-packages/dask/dataframe/core.pyt__init__Ms   $ cC`s|jS(N(Ra(Rg((s2lib/python2.7/site-packages/dask/dataframe/core.pyt__dask_graph__ZscC`s |jgS(N(tkey(Rg((s2lib/python2.7/site-packages/dask/dataframe/core.pyt __dask_keys__]scC`s|jS(N(Rb(Rg((s2lib/python2.7/site-packages/dask/dataframe/core.pyt__dask_tokenize__`scC`s |jfS(N(Rn(Rg((s2lib/python2.7/site-packages/dask/dataframe/core.pyt__dask_layers__csRntdataframe_optimizetfalseycC`s tdfS(N((R (Rg((s2lib/python2.7/site-packages/dask/dataframe/core.pyt__dask_postcompute__jscC`st|j|j|jffS(N(R^RbRfRk(Rg((s2lib/python2.7/site-packages/dask/dataframe/core.pyt__dask_postpersist__mscC`s|jS(N(Rf(Rg((s2lib/python2.7/site-packages/dask/dataframe/core.pyt_meta_nonemptypscC`s |jjS(N(Rftdtype(Rg((s2lib/python2.7/site-packages/dask/dataframe/core.pyRwtscC`sTttt|}|j|jt|jdsJ|jdnt|S(NRw( tsettdirRetupdatet__dict__thasattrRfRtlist(Rgto((s2lib/python2.7/site-packages/dask/dataframe/core.pyt__dir__xs cC`s ddgS(s6Dummy divisions to be compat with Series and DataFrameN(tNone(Rg((s2lib/python2.7/site-packages/dask/dataframe/core.pyRkscC`sxt|jdkr|jn|jd d}t|jdrTd|jj}ndt|jj}d||fS(Ni is...Rws , dtype=%ss , type=%ssdd.Scalar<%s%s>(RURbR|RfRwRet__name__(RgRitextra((s2lib/python2.7/site-packages/dask/dataframe/core.pyt__repr__s /cC`stj|jS(N(RMtasarraytcompute(Rg((s2lib/python2.7/site-packages/dask/dataframe/core.pyt __array__scC`s|j|j|jfS(N(RaRbRf(Rg((s2lib/python2.7/site-packages/dask/dataframe/core.pyt_argsscC`s|jS(N(R(Rg((s2lib/python2.7/site-packages/dask/dataframe/core.pyt __getstate__scC`s|\|_|_|_dS(N(RaRbRf(Rgtstate((s2lib/python2.7/site-packages/dask/dataframe/core.pyt __setstate__scC`s |jdfS(Ni(Rb(Rg((s2lib/python2.7/site-packages/dask/dataframe/core.pyRnsc`sfd}|S(Nc`svtdt|}i|jdff|df6}|j}tj||d|g}t|||S(Nt-iR_(R!R/RbRvR5R`R^(RgRiRhRjtgraph(top(s2lib/python2.7/site-packages/dask/dataframe/core.pytfs "((tclsRR((Rs2lib/python2.7/site-packages/dask/dataframe/core.pyt_get_unary_operatorsc`sfdS(Nc`st||dS(Ntinv(t_scalar_binary(Rgtother(RR(s2lib/python2.7/site-packages/dask/dataframe/core.pytt((RRR((RRs2lib/python2.7/site-packages/dask/dataframe/core.pyt_get_binary_operatorscC`sb|j}|rR|j||j}d|j}tj||dd}nt|j|S(sConvert into a ``dask.delayed`` object. Parameters ---------- optimize_graph : bool, optional If True [default], the graph is optimized before converting into ``dask.delayed`` objects. sdelayed-R_((Rmt__dask_optimize__RoRbR5R`R3Rn(Rgtoptimize_graphRhRi((s2lib/python2.7/site-packages/dask/dataframe/core.pyt to_delayeds  N($Rt __module__t__doc__RRlRmRoRpRqRR;R0Rt staticmethodRtgett__dask_scheduler__RtRutpropertyRvRwRRkRRRRRRnt classmethodRtFalseRRWR(((s2lib/python2.7/site-packages/dask/dataframe/core.pyR^Ks2             c C`sdjt|t||}|g}i}t|}t|trj|j||jdf}nt|rzt S|}|r|j i|||jdff|df6n,|j i||jdf|f|df6t |} t | } |r|| |j } n||j | } tj||d|} |tk ru|| || |jj|jjgSt| || SdS(Ns{0}-{1}iR_(RdR!R/tget_parallel_typeRKR^tappendRbR1tNotImplementedRzR=R<RvR5R`tindextmintmax( RRgRRRiR_Rht return_typet other_keyt other_metatother_meta_nonemptyRjR((s2lib/python2.7/site-packages/dask/dataframe/core.pyRs.!    /,    t_Framec B`seZdZdZdZdZdZdZee ddde Z e e jZd Zd Zed Zed Zed ZedZedZdZdZdZdsdZdsdZdZedZdZ edZ!dZ"edZ#e#j$dZ#e%dZ&edZ'dZ(dZ)e*e+j,dsd d!Z-d"Z.d#Z/e/Z0d$Z1d%Z2d&Z3e3Z4d'Z5e6d(d)d*Z7e6d(d)d+Z8e6d(d)dsdse9dsdsdsdsdsd,Z:e*e+j,d-Z;dsd.Z<d/d e=d0Z>d/e=d1Z?ed2Z@d3ZAed4ZBdsdsdse%d5ZCe*e+j,dsdsdsdsd6ZDe*e+j,dsdsd7ZEe*e+j,dsdsd8ZFdsdse%dsd9ZGe*e+j,dsdse%d:ZHdsd;ZId<e%d=ZJd>ZKd?ZLe=d@ZMeNdAZOeNe%dBZPdsdse%dsdCdDZQe*e+j,d dCdEZRe*e+j,d dsdCdFZSdse=e%dsdGZTe*e+j,dHZUe*e+j,dse=e%dsdIZVe*e+j,dse=e%dsdJZWe*e+j,dse=e%dsdsdsdKZXe*e+j,dse=e%dsdsdsdLZYe*e+j,dse=e%dsdMZZe*e+j,dse=e%dsdNZ[e*e+j,dse=e%dOZ\e*e+j,dse=e%dPZ]e*e+j,dse%dQZ^e*e+j,dse=e%dsdsdRZ_e*e+j,dse=d e%dsdsdSZ`e*e+j,dse=d e%dsdsdTZae*e+j,dsdsd e%dUZbdVdCdWdXZce*e+j,e%dsdWdYZde=dsdsdZZee*e+j,dse=dsdsd[Zfe*e+j,dse=dsdsd\Zge*e+j,dse=dsd]Zhe*e+j,dse=dsd^Zie*e+j,ejjkd_Zle*e+j,ejjkd`Zme*e+j,daZne*e+j,dbZoe*e+j,dcZpe*e+j,ddZqe*e+j,deZre*e+jse%dfZte*e+j,dgdsdsdhZue*e+j,dse=diZve*e+j,djZweNdkZxe*e+j,dsdsdlZye*e+j,dmZze*e+j,dnZ{dsdoZ|edpZ}dqZ~drZRS(ts  Superclass for DataFrame and Series Parameters ---------- dsk: dict The dask graph to compute this DataFrame name: str The key prefix that specifies which keys in the dask comprise this particular DataFrame / Series meta: pandas.DataFrame, pandas.Series, or pandas.Index An empty pandas object with names, dtypes, and indices matching the expected output. divisions: tuple of index values Values along which we partition our blocks on the index cC`st|ts*tj||dg}n||_||_t|}|j|stdjt |j t t |n||_ t ||_dS(NR_s/Expected meta to specify type {0}, got type {1}(RKR5R`RaRbR=t_is_partition_typeRcRdReRR(RfttupleRk(RgRhRiRjRk((s2lib/python2.7/site-packages/dask/dataframe/core.pyRls      cC`s|jS(N(Ra(Rg((s2lib/python2.7/site-packages/dask/dataframe/core.pyRmscC`s)gt|jD]}|j|f^qS(N(tranget npartitionsRb(RgRY((s2lib/python2.7/site-packages/dask/dataframe/core.pyRoscC`s |jfS(N(Rb(Rg((s2lib/python2.7/site-packages/dask/dataframe/core.pyRqscC`s|jS(N(Rb(Rg((s2lib/python2.7/site-packages/dask/dataframe/core.pyRpsRnRrRscC`s tdfS(N((R](Rg((s2lib/python2.7/site-packages/dask/dataframe/core.pyRtscC`s"t||j|j|jffS(N(ReRbRfRk(Rg((s2lib/python2.7/site-packages/dask/dataframe/core.pyRuscC`stS(N(t new_dd_object(Rg((s2lib/python2.7/site-packages/dask/dataframe/core.pyt _constructorscC`st|jdS(sReturn number of partitionsi(RURk(Rg((s2lib/python2.7/site-packages/dask/dataframe/core.pyRsc C`s(|jtjtjdddtdtS(sSize of the Series or DataFrame as a Delayed object. Examples -------- >>> series.size # doctest: +SKIP dd.Scalar ttokentsizeRjt split_every(t reductionR6RRMtsumtintR(Rg((s2lib/python2.7/site-packages/dask/dataframe/core.pyRs !cC`s t|jS(s/ A non-empty version of `_meta` with fake data.(R<Rf(Rg((s2lib/python2.7/site-packages/dask/dataframe/core.pyRv)scC`s|j|j|j|jfS(N(RaRbRfRk(Rg((s2lib/python2.7/site-packages/dask/dataframe/core.pyR.scC`s|jS(N(R(Rg((s2lib/python2.7/site-packages/dask/dataframe/core.pyR2scC`s"|\|_|_|_|_dS(N(RaRbRfRk(RgR((s2lib/python2.7/site-packages/dask/dataframe/core.pyR5scC`st|j|j|j|jS(s Make a copy of the dataframe This is strictly a shallow copy of the underlying computational graph. It does not affect the underlying data (RRaRbRfRk(Rg((s2lib/python2.7/site-packages/dask/dataframe/core.pytcopy8scK`s%|j|_tj|j}|S(N(Rt _computedRMR(RgRwtkwargstx((s2lib/python2.7/site-packages/dask/dataframe/core.pyRAscC`s tdS(N(tNotImplementedError(RgRtcontext((s2lib/python2.7/site-packages/dask/dataframe/core.pyt__array_wrap__Fsc O`s|jdd}xj||D]^}t|tjrJ|jdkrJqqt|ttttt j t j t j fst SqW|dkr|jdk rt S|jdkrt St|||Snt SdS(Ntoutt__call__i(((RRKRMRNtshapeRR^RR*RRt DataFrameRStIndexRt signatureRtnouttelemwise(Rgt numpy_ufunctmethodtinputsRRR((s2lib/python2.7/site-packages/dask/dataframe/core.pyt__array_ufunc__Is! cC`stS(N(R(Rg((s2lib/python2.7/site-packages/dask/dataframe/core.pyt _elemwiseascC`s tdS(N(R(Rg((s2lib/python2.7/site-packages/dask/dataframe/core.pyt _repr_dataescC`s]dj|j}|jr6tj|jd|}n#tjdg|jdd|}|S(Nsnpartitions={0}RiRi(RdRtknown_divisionsRRRRk(RgRiRk((s2lib/python2.7/site-packages/dask/dataframe/core.pyt_repr_divisionshs  #c C`sX|jjdddt}djd|jjd|dt|jdt|j S( Ntmax_rowsitshow_dimensionss>Dask {klass} Structure: {data} Dask Name: {name}, {task} taskstklasstdataRittask( Rt to_stringRRdt __class__RR%RbRURa(RgR((s2lib/python2.7/site-packages/dask/dataframe/core.pyRrscC`s)|jtdd|jdd|jjS(sReturn dask Index instanceRRs-indexRj(tmap_partitionstgetattrRbRfR(Rg((s2lib/python2.7/site-packages/dask/dataframe/core.pyRzscC`sI|j|_ttj||}|j|_|j|_|j|_dS(N(RkRR6t assign_indexRaRbRf(Rgtvaluetresult((s2lib/python2.7/site-packages/dask/dataframe/core.pyRs    cC`s|jtjd|jS(syReset the index to the default index. Note that unlike in ``pandas``, the reset ``dask.dataframe`` index will not be monotonically increasing from 0. Instead, it will restart at 0 for each partition (e.g. ``index1 = [0, ..., 10], index2 = [0, ...]``). This is due to the inability to statically know the full length of the index. For DataFrame with multi-level index, returns a new DataFrame with labeling information in the columns under the index names, defaulting to 'level_0', 'level_1', etc. if any are None. For a standard index, the index name will be used (if set), otherwise a default 'index' or 'level_0' (if 'index' is already taken) will be used. Parameters ---------- drop : boolean, default False Do not try to insert index into dataframe columns. tdrop(RR$t reset_indextclear_divisions(RgR((s2lib/python2.7/site-packages/dask/dataframe/core.pyRscC`s&t|jdko%|jddk S(s#Whether divisions are already knowniN(RURkR(Rg((s2lib/python2.7/site-packages/dask/dataframe/core.pyRscC`s3d|jd}t||j|j|j|S(s Forget division information iN(N(RRReRaRbRf(RgRk((s2lib/python2.7/site-packages/dask/dataframe/core.pyRscC`sd|ko|jknrdt||jf}|j||d!}i|j|f|df6}tj||d|g}t|||j|Sdj|j}t |dS(s=Get a dask DataFrame/Series representing the `nth` partition.isget-partition-%s-%siR_sn must be 0 <= n < {0}N( RtstrRbRkR5R`RRfRdt ValueError(RgtnRiRktlayerRtmsg((s2lib/python2.7/site-packages/dask/dataframe/core.pyt get_partitionsicK`s|jj|d|krF|ddk rFt}i|dd6}n d}}|jdttkrwtdntj}t |d|d|d|j dd d |d |d |d ||S(Ntsubsettcolstkeepsdrop_duplicates with keep=Falsetchunkt aggregateRjRsdrop-duplicatesRt split_outtsplit_out_setuptsplit_out_setup_kwargs( Rvtdrop_duplicatesRtsplit_out_on_colsRRWRRR$tacaRf(RgRRRRRR((s2lib/python2.7/site-packages/dask/dataframe/core.pyRs    c C`s+|jttjdddtdtjS(NRRURjR(RRURMRRRR(Rg((s2lib/python2.7/site-packages/dask/dataframe/core.pyt__len__scC`stdj|jjdS(Ns>The truth value of a {0} is ambiguous. Use a.any() or a.all().(RRdRR(Rg((s2lib/python2.7/site-packages/dask/dataframe/core.pyt__bool__s c`sfd}|S(Nc`stdjtdS(Ns cannot convert the series to {0}(RcRdR((t cast_type(s2lib/python2.7/site-packages/dask/dataframe/core.pytwrappers ((RgRR((Rs2lib/python2.7/site-packages/dask/dataframe/core.pyt _scalarfuncscC`s |jtS(N(Rtfloat(Rg((s2lib/python2.7/site-packages/dask/dataframe/core.pyt __float__scC`s |jtS(N(RR(Rg((s2lib/python2.7/site-packages/dask/dataframe/core.pyt__int__scC`s |jtS(N(Rtcomplex(Rg((s2lib/python2.7/site-packages/dask/dataframe/core.pyt __complex__stpadi cO`st||||S(s Apply Python function on each DataFrame partition. Note that the index and divisions are assumed to remain unchanged. Parameters ---------- func : function Function applied to each partition. args, kwargs : Arguments and keywords to pass to the function. The partition will be the first argument, and these will be passed *after*. Arguments and keywords may contain ``Scalar``, ``Delayed`` or regular python objects. DataFrame-like args (both dask and pandas) will be repartitioned to align (if necessary) before applying the function. $META Examples -------- Given a DataFrame, Series, or Index, such as: >>> import dask.dataframe as dd >>> df = pd.DataFrame({'x': [1, 2, 3, 4, 5], ... 'y': [1., 2., 3., 4., 5.]}) >>> ddf = dd.from_pandas(df, npartitions=2) One can use ``map_partitions`` to apply a function on each partition. Extra arguments and keywords can optionally be provided, and will be passed to the function after the partition. Here we apply a function with arguments and keywords to a DataFrame, resulting in a Series: >>> def myadd(df, a, b=1): ... return df.x + df.y + a + b >>> res = ddf.map_partitions(myadd, 1, b=2) >>> res.dtype dtype('float64') By default, dask tries to infer the output metadata by running your provided function on some fake data. This works well in many cases, but can sometimes be expensive, or even fail. To avoid this, you can manually specify the output metadata with the ``meta`` keyword. This can be specified in many forms, for more information see ``dask.dataframe.utils.make_meta``. Here we specify the output is a Series with no name, and dtype ``float64``: >>> res = ddf.map_partitions(myadd, 1, b=2, meta=(None, 'f8')) Here we map a function that takes in a DataFrame, and returns a DataFrame with a new column: >>> res = ddf.map_partitions(lambda df: df.assign(z=df.x * df.y)) >>> res.dtypes x int64 y float64 z float64 dtype: object As before, the output metadata can also be specified manually. This time we pass in a ``dict``, as the output is a DataFrame: >>> res = ddf.map_partitions(lambda df: df.assign(z=df.x * df.y), ... meta={'x': 'i8', 'y': 'f8', 'z': 'f8'}) In the case where the metadata doesn't change, you can also pass in the object itself directly: >>> res = ddf.map_partitions(lambda df: df.head(), meta=df) Also note that the index and divisions are assumed to remain unchanged. If the function you're mapping changes the index/divisions, you'll need to clear them afterwards: >>> ddf.map_partitions(func).clear_divisions() # doctest: +SKIP (R(RgtfuncRXR((s2lib/python2.7/site-packages/dask/dataframe/core.pyRsOcO`s)ddlm}|||||||S(sApply a function to each partition, sharing rows with adjacent partitions. This can be useful for implementing windowing functions such as ``df.rolling(...).mean()`` or ``df.diff()``. Parameters ---------- func : function Function applied to each partition. before : int The number of rows to prepend to partition ``i`` from the end of partition ``i - 1``. after : int The number of rows to append to partition ``i`` from the beginning of partition ``i + 1``. args, kwargs : Arguments and keywords to pass to the function. The partition will be the first argument, and these will be passed *after*. $META Notes ----- Given positive integers ``before`` and ``after``, and a function ``func``, ``map_overlap`` does the following: 1. Prepend ``before`` rows to each partition ``i`` from the end of partition ``i - 1``. The first partition has no rows prepended. 2. Append ``after`` rows to each partition ``i`` from the beginning of partition ``i + 1``. The last partition has no rows appended. 3. Apply ``func`` to each partition, passing in any extra ``args`` and ``kwargs`` if provided. 4. Trim ``before`` rows from the beginning of all but the first partition. 5. Trim ``after`` rows from the end of all but the last partition. Note that the index and divisions are assumed to remain unchanged. Examples -------- Given a DataFrame, Series, or Index, such as: >>> import dask.dataframe as dd >>> df = pd.DataFrame({'x': [1, 2, 4, 7, 11], ... 'y': [1., 2., 3., 4., 5.]}) >>> ddf = dd.from_pandas(df, npartitions=2) A rolling sum with a trailing moving window of size 2 can be computed by overlapping 2 rows before each partition, and then mapping calls to ``df.rolling(2).sum()``: >>> ddf.compute() x y 0 1 1.0 1 2 2.0 2 4 3.0 3 7 4.0 4 11 5.0 >>> ddf.map_overlap(lambda df: df.rolling(2).sum(), 2, 0).compute() x y 0 NaN NaN 1 3.0 3.0 2 6.0 5.0 3 11.0 7.0 4 18.0 9.0 The pandas ``diff`` method computes a discrete difference shifted by a number of periods (can be positive or negative). This can be implemented by mapping calls to ``df.diff`` to each partition after prepending/appending that many rows, depending on sign: >>> def diff(df, periods=1): ... before, after = (periods, 0) if periods > 0 else (0, -periods) ... return df.map_overlap(lambda df, periods=1: df.diff(periods), ... periods, 0, periods=periods) >>> diff(ddf, 1).compute() x y 0 NaN NaN 1 1.0 1.0 2 2.0 1.0 3 3.0 1.0 4 4.0 1.0 If you have a ``DatetimeIndex``, you can use a ``pd.Timedelta`` for time- based windows. >>> ts = pd.Series(range(10), index=pd.date_range('2017', periods=10)) >>> dts = dd.from_pandas(ts, npartitions=2) >>> dts.map_overlap(lambda df: df.rolling('2D').sum(), ... pd.Timedelta('2D'), 0).compute() 2017-01-01 0.0 2017-01-02 1.0 2017-01-03 3.0 2017-01-04 5.0 2017-01-05 7.0 2017-01-06 9.0 2017-01-07 11.0 2017-01-08 13.0 2017-01-09 15.0 2017-01-10 17.0 dtype: float64 i(t map_overlap(trollingR(RgRtbeforetafterRXRR((s2lib/python2.7/site-packages/dask/dataframe/core.pyR5skc K`s|dkr|}n|dkrE| r6tdn|}|} n|rW|jni}||d<| ry| jni} || d<|r|jni}||d>> import pandas as pd >>> import dask.dataframe as dd >>> df = pd.DataFrame({'x': range(50), 'y': range(50, 100)}) >>> ddf = dd.from_pandas(df, npartitions=4) Count the number of rows in a DataFrame. To do this, count the number of rows in each partition, then sum the results: >>> res = ddf.reduction(lambda x: x.count(), ... aggregate=lambda x: x.sum()) >>> res.compute() x 50 y 50 dtype: int64 Count the number of rows in a Series with elements greater than or equal to a value (provided via a keyword). >>> def count_greater(x, value=0): ... return (x >= value).sum() >>> res = ddf.x.reduction(count_greater, aggregate=lambda x: x.sum(), ... chunk_kwargs={'value': 25}) >>> res.compute() 25 Aggregate both the sum and count of a Series at the same time: >>> def sum_and_count(x): ... return pd.Series({'count': x.count(), 'sum': x.sum()}, ... index=['count', 'sum']) >>> res = ddf.x.reduction(sum_and_count, aggregate=lambda x: x.sum()) >>> res.compute() count 50 sum 1225 dtype: int64 Doing the same, but for a DataFrame. Here ``chunk`` returns a DataFrame, meaning the input to ``aggregate`` is a DataFrame with an index with non-unique entries for both 'x' and 'y'. We groupby the index, and sum each group to get the final result. >>> def sum_and_count(x): ... return pd.DataFrame({'count': x.count(), 'sum': x.sum()}, ... columns=['count', 'sum']) >>> res = ddf.reduction(sum_and_count, ... aggregate=lambda x: x.groupby(level=0).sum()) >>> res.compute() count sum x 50 1225 y 50 3725 s+`combine_kwargs` provided with no `combine`t aca_chunkt aca_combinet aca_aggregateRRtcombineRjRRt chunk_kwargstaggregate_kwargstcombine_kwargsN(RRRRt_reduction_chunkt_reduction_aggregatet_reduction_combine( RgRRRRjRRRRRR((s2lib/python2.7/site-packages/dask/dataframe/core.pyRs$g        cO`set|trQ|\}}||kr:td|n|||<|||S||||SdS(Ns1%s is both the pipe target and a keyword argument(RKRR(RgRRXRttarget((s2lib/python2.7/site-packages/dask/dataframe/core.pytpipe"s     c `s%tjtds'tdntj|}t|}d|fdt|D}g}xtt D]d|ffdtjD}t j t ||dg}t |jj} |j| qW|S(s Pseudorandomly split dataframe into different pieces row-wise Parameters ---------- frac : list List of floats that should sum to one. random_state: int or np.random.RandomState If int create a new RandomState with this as the seed Otherwise draw from the passed RandomState Examples -------- 50/50 split >>> a, b = df.random_split([0.5, 0.5]) # doctest: +SKIP 80/10/10 split, consistent random_state >>> a, b, c = df.random_split([0.8, 0.1, 0.1], random_state=123) # doctest: +SKIP See Also -------- dask.DataFrame.sample isfrac should sum to 1ssplit-c`s:i|]0\}}tj|f|f|fqS((tpd_splitRb(t.0RYR(tfracRiRg(s2lib/python2.7/site-packages/dask/dataframe/core.pys Os s split-%d-%sc`s.i|]$}t|ff|fqS((R(R tj(RYRitname2(s2lib/python2.7/site-packages/dask/dataframe/core.pys Us R_(RMtallcloseRRRRR/t enumerateRRUR5R`R ReRfRkR( RgRt random_statet state_dataRRRtdsk2Rtout_df((RRYRiRRgs2lib/python2.7/site-packages/dask/dataframe/core.pyt random_split0s  $!ic C`s|dkr|j}n||jkrKd}t|j|j|nd|||jf}|dkr d||jf}i}x9t|D]+}tj|j|f|f|||f>> df.loc["b"] # doctest: +SKIP >>> df.loc["b":"d"] # doctest: +SKIP i(t _LocIndexer(tindexingR(RgR((s2lib/python2.7/site-packages/dask/dataframe/core.pytlocsc `st|ts|f}nddlm}|||jf}td|D}dt||tj|jdt |j }g|D]\}}|j |^q|j |dddg}fdt |D}t j|d |g}t||j|S( Ni(tnormalize_indexcs`s7|]-}t|tr+t||dn|VqdS(iN(RKRtslice(R tk((s2lib/python2.7/site-packages/dask/dataframe/core.pys ssblocks-Rwiic`s+i|]!\}}t||fqS((R(R RYRn(Ri(s2lib/python2.7/site-packages/dask/dataframe/core.pys s R_(RKRt array.slicingRRR/RMRRotobjectttolistRkRR5R`RRf( RgRRtnew_keyst_RYRkRhR((Ris2lib/python2.7/site-packages/dask/dataframe/core.pyt _partitionss   %@cC`s t|jS(sG Slice dataframe by partitions This allows partitionwise slicing of a Dask Dataframe. You can perform normal Numpy-style slicing but now rather than slice elements of the array you slice along partitions so, for example, ``df.partitions[:5]`` produces a new Dask Dataframe of the first five partitions. Examples -------- >>> df.partitions[0] # doctest: +SKIP >>> df.partitions[:3] # doctest: +SKIP >>> df.partitions[::10] # doctest: +SKIP Returns ------- A Dask DataFrame (RR'(Rg((s2lib/python2.7/site-packages/dask/dataframe/core.pyt partitionsscC`s|dk r(|dk r(tjdn|dk rAt||S|dk r`t||d|S|dk r|t|d|StddS(s Repartition dataframe along new divisions Parameters ---------- divisions : list, optional List of partitions to be used. If specified npartitions will be ignored. npartitions : int, optional Number of partitions of output. Only used if divisions isn't specified. freq : str, pd.Timedelta A period on which to partition timeseries data like ``'7D'`` or ``'12h'`` or ``pd.Timedelta(hours=12)``. Assumes a datetime index. force : bool, default False Allows the expansion of the existing divisions. If False then the new divisions lower and upper bounds must be the same as the old divisions. Examples -------- >>> df = df.repartition(npartitions=10) # doctest: +SKIP >>> df = df.repartition(divisions=[0, 5, 10, 20]) # doctest: +SKIP >>> df = df.repartition(freq='7d') # doctest: +SKIP sVWhen providing both npartitions and divisions to repartition only npartitions is used.tforcetfreqs8Provide either divisions= or npartitions= to repartitionN(Rtwarningstwarntrepartition_npartitionst repartitiontrepartition_freqR(RgRkRR*R)((s2lib/python2.7/site-packages/dask/dataframe/core.pyR.s    c `s j|}dkr6|dk r6tdnt|trX|jjd}n|}jjd|dd|d|}|dksdkr t|rt | rd}i|d6}n|f}i}j t jdd|d|d|||SdkrCd d|dkr3dn|d} } n2d j dd|dkrkdn|} } |dkrd t fd tj D} tj| dg} t| |j} n} | jt j| | dd|d|S(Ns%fillna with set limit and method=NoneiRRtlimittaxisiRjRtffilltbfills fillna-chunk-c`s=i|]3}tjj|f|kf|fqS((R6t fillna_checkRb(R RY(RRiRgt skip_check(s2lib/python2.7/site-packages/dask/dataframe/core.pys s R_((RR2(t_validate_axisRRRKRRvtvaluestfillnaRFR1RR$RR/RR5R`RRkR(RgRRR0R1t test_valueRjRXRRRRhRtparts((RRiRgR5s2lib/python2.7/site-packages/dask/dataframe/core.pyR8sB  "  cC`s|jddd|d|S(NRR2R0R1(R8(RgR1R0((s2lib/python2.7/site-packages/dask/dataframe/core.pyR2scC`s|jddd|d|S(NRR3R0R1(R8(RgR1R0((s2lib/python2.7/site-packages/dask/dataframe/core.pyR3"sc `s|dk rbd}t|trSd|ko8dknrStj||qbt|ndkr}tdn|dkrtjj}ndt |t j |}fdt |D}t j|dg}t|jjS( s Random sample of items Parameters ---------- n : int, optional Number of items to return is not supported by dask. Use frac instead. frac : float, optional Fraction of axis items to return. replace : boolean, optional Sample with or without replacement. Default = False. random_state : int or ``np.random.RandomState`` If int we create a new RandomState with this as the seed Otherwise we draw from the passed RandomState See Also -------- DataFrame.random_split pandas.DataFrame.sample slsample does not support the number of sampled items parameter, 'n'. Please use the 'frac' parameter instead.iisfrac must not be Nonessample-c`s@i|]6\}}tjj|f|f|fqS((R6tsampleRb(R RYR(RRitreplaceRg(s2lib/python2.7/site-packages/dask/dataframe/core.pys Ms R_N(RRKRR+R,RRMtrandomt RandomStateR/RRRR5R`RRfRk( RgRRR<RRRRhR((RRiR<Rgs2lib/python2.7/site-packages/dask/dataframe/core.pyR;&s  +    cC`s"|jtjd|d|d|S(Nt to_replaceRtregex(RR$R<(RgR?RR@((s2lib/python2.7/site-packages/dask/dataframe/core.pyR<SscC`sddlm}|tkr:t|jtj}n|jtj}t |t rt|}t||j krt dj t||j n|jdkr||f}n||t|jff}||_n$|dk rt dj |n|S(sConvert a dask DataFrame to a dask array. Parameters ---------- lengths : bool or Sequence of ints, optional How to determine the chunks sizes for the output array. By default, the output array will have unknown chunk lengths along the first axis, which can cause some later operations to fail. * True : immediately compute the length of each partition * Sequence : a sequence of integers to use for the chunk sizes on the first axis. These values are *not* validated for correctness, beyond ensuring that the number of items matches the number of partitions. Returns ------- i(tnormalize_chunkssRThe number of items in 'lengths' does not match the number of partitions. {} != {}is$Unexpected value for 'lengths': '{}'N(tdask.array.coreRARWRRRURRMRRKRRRRdtndimtcolumnst_chunksR(RgtlengthsRAtarrtchunks((s2lib/python2.7/site-packages/dask/dataframe/core.pyt to_dask_arrayXs"    tacK`s)ddlm}|||||||S(s. See dd.to_hdf docstring for more information i(tto_hdf(tioRK(Rgt path_or_bufRntmodeRRRK((s2lib/python2.7/site-packages/dask/dataframe/core.pyRKscK`s ddlm}||||S(s. See dd.to_csv docstring for more information i(tto_csv(RLRO(RgtfilenameRRO((s2lib/python2.7/site-packages/dask/dataframe/core.pyROscO`s#ddlm}|||||S(s/ See dd.to_json docstring for more information i(tto_json(RLRQ(RgRPRXRRQ((s2lib/python2.7/site-packages/dask/dataframe/core.pyRQscC`s~|j}|j}|r^|j||j}d|j}tj||dd}ng|D]}t||^qeS(sConvert into a list of ``dask.delayed`` objects, one per partition. Parameters ---------- optimize_graph : bool, optional If True [default], the graph is optimized before converting into ``dask.delayed`` objects. Examples -------- >>> partitions = df.to_delayed() # doctest: +SKIP See Also -------- dask.dataframe.from_delayed sdelayed-R_((RoRmRRbR5R`R3(RgRtkeysRRiR!((s2lib/python2.7/site-packages/dask/dataframe/core.pyRs   c`s fdS(Nc`s t|S(N(R(Rg(R(s2lib/python2.7/site-packages/dask/dataframe/core.pyRR((RR((Rs2lib/python2.7/site-packages/dask/dataframe/core.pyRsc`s$|rfdSfdSdS(Nc`st||S(N(R(RgR(R(s2lib/python2.7/site-packages/dask/dataframe/core.pyRRc`st||S(N(R(RgR(R(s2lib/python2.7/site-packages/dask/dataframe/core.pyRR((RRR((Rs2lib/python2.7/site-packages/dask/dataframe/core.pyRs icC`sddlm}t|tr=|dkr=tdq=n|d k rt|tsgtdn|dkrtdqn||d|d|d|d |d |d |S( siProvides rolling transformations. Parameters ---------- window : int, str, offset Size of the moving window. This is the number of observations used for calculating the statistic. The window size must not be so large as to span more than one adjacent partition. If using an offset or offset alias like '5D', the data must have a ``DatetimeIndex`` .. versionchanged:: 0.15.0 Now accepts offsets and string offset aliases min_periods : int, default None Minimum number of observations in window required to have a value (otherwise result is NA). center : boolean, default False Set the labels at the center of the window. win_type : string, default None Provide a window type. The recognized window types are identical to pandas. axis : int, default 0 Returns ------- a Rolling object on which to call a method to compute a statistic Notes ----- The `freq` argument is not supported. i(tRollingswindow must be >= 0smin_periods must be an integersmin_periods must be >= 0twindowt min_periodsR*tcentertwin_typeR1N(tdask.dataframe.rollingRSRKRRR(RgRTRUR*RVRWR1RS((s2lib/python2.7/site-packages/dask/dataframe/core.pyRs"   cC`s|j|}t|ts-tdn|dkr[|jtjddd|ddS|dkrs|dfn d| f\}}|jtj||ddd|S(Nsperiods must be an integeriRtdifftperiodsR1i(R6RKRRcRR$RYR(RgRZR1RR((s2lib/python2.7/site-packages/dask/dataframe/core.pyRYs +c C`s|j|}t|ts-tdn|dkra|jtjddd|d|ddS|dkr|dkr|dfn d| f\}}|jtj||ddd|S|j j|d|}|jtjddd|d|d |d t }t ||d|S( Nsperiods must be an integeriRtshiftRZR*R1iRjttransform_divisions( R6RKRRcRR$R[RRRvRtmaybe_shift_divisions(RgRZR*R1RRRjR((s2lib/python2.7/site-packages/dask/dataframe/core.pyR[s   +  c C`s|j|}t|j|d|d|}|j|}tt|}|dkr|j|d|d|d|d|} t|| S|j|d|d|d|d|d|} t|t rt |j t |j f| _ nt|| SdS(NR1tskipnaiRjRR(R6RRvt _token_prefixR$Rt handle_outRRKRRRDRRk( RgRiR1R^RRRjRRR((s2lib/python2.7/site-packages/dask/dataframe/core.pyt_reduction_aggs!     $cC`s2t|d|jj}|jtjd|S(NtabsRj(t_raise_if_object_seriesRvRbRR$(RgRj((s2lib/python2.7/site-packages/dask/dataframe/core.pyRb#s c C`s%|jdd|d|d|d|S(NtallR1R^RR(Ra(RgR1R^RR((s2lib/python2.7/site-packages/dask/dataframe/core.pyRd)sc C`s%|jdd|d|d|d|S(NtanyR1R^RR(Ra(RgR1R^RR((s2lib/python2.7/site-packages/dask/dataframe/core.pyRe.sc C`sc|jdd|d|d|d|}|r[|j|jjd||kdtjS|SdS(NRR1R^RRR(RatwheretnotnullRRMtNaN(RgR1R^RRwRt min_countR((s2lib/python2.7/site-packages/dask/dataframe/core.pyR3s $ c C`sc|jdd|d|d|d|}|r[|j|jjd||kdtjS|SdS(NtprodR1R^RRR(RaRfRgRRMRh(RgR1R^RRwRRiR((s2lib/python2.7/site-packages/dask/dataframe/core.pyRj>s $ c C`s%|jdd|d|d|d|S(NRR1R^RR(Ra(RgR1R^RR((s2lib/python2.7/site-packages/dask/dataframe/core.pyRIsc C`s%|jdd|d|d|d|S(NRR1R^RR(Ra(RgR1R^RR((s2lib/python2.7/site-packages/dask/dataframe/core.pyRNscC`sd}|j|}|jjd|d|}|dkrkttj|d|d|j|d|d|St| }t|gdtdt d t d|d i|d 6d|j|d |d|d | }t |t rt |jt|jf|_n|SdS(NtidxmaxR1R^iRjRRRRRtscalarRtfn(R6RvRkRR$R_RFRtidxmaxmin_chunkt idxmaxmin_aggtidxmaxmin_combineRKRRRDRRk(RgR1R^RRmRjRlR((s2lib/python2.7/site-packages/dask/dataframe/core.pyRkSs       $cC`sd}|j|}|jjd|}|dkrettj|d|d|j|d|d|St| }t|gdt dt d t d|d i|d 6d|j|d |d|d | }t |t rt|jt|jf|_n|SdS(NtidxminR1iRjRR^RRRRRlRRm(R6RvRkRR$RqR_RFRRnRoRpRKRRRDRRk(RgR1R^RRmRjRlR((s2lib/python2.7/site-packages/dask/dataframe/core.pyRqgs       $c C`s|j|}|jd}|dkr_|jjd|}|jtjd|d|d|S|jj}|jtjdtjd|d|d|}t|t rt |j t |j f|_ n|SdS(NtcountiR1RjRRR(R6R_RvRrRR$RRRKRRRDRRk(RgR1RRRjR((s2lib/python2.7/site-packages/dask/dataframe/core.pyRr{s  $c C`s9|j|}t|d|jjd|d|}|dkrttj|d|d|jdd|d|}t||S|j}|j d|d|} |j d|} |jdt |||} tt j | | d| d|}t|tr(t|jt|jf|_nt||SdS( NtmeanR1R^iRjRRsmean-%s(R6RcRvRsRR$R_R`t_get_numeric_dataRRrR/R6tmean_aggregateRKRRRDRRk( RgR1R^RRwRRjRtnumtsRRi((s2lib/python2.7/site-packages/dask/dataframe/core.pyRss"     $c C`s\|j|}t|d|jjd|d|}|dkrttj|d|d|jdd|d|d|}t||S|j} d| j d|d |} d| d j d|d |} | j d |} |jd} tt j | | | d| d|d|}t |trKt|jt|jf|_nt||SdS( NtvarR1R^iRjRtddofg?Ri(R6RcRvRxRR$R_R`RtRRrR6t var_aggregateRKRRRDRRk(RgR1R^RyRRwRRjRRvRtx2RRi((s2lib/python2.7/site-packages/dask/dataframe/core.pyRxs$       $c C`s|j|}t|d|jjd|d|}|dkrttj|d|d|jdd|d|d|}t||S|jd|d|d|} |jd} tt j | d|d| }t||SdS( NtstdR1R^iRjRRyR( R6RcRvR|RR$R_R`RxRMtsqrt( RgR1R^RyRRwRRjRtvRi((s2lib/python2.7/site-packages/dask/dataframe/core.pyR|s     c C`s$|j|}t|d|jjd|d|d|}|dkr~ttj|d|d|jdd|d|d|S|j}|jd|d|d|}|j d|}|jd} tt j ||d|d| } t |t rt|jt|jf| _n| SdS( NtsemR1R^RyiRjRR(R6RcRvRRR$R_RtRxRrRMR}RKRRRDRRk( RgR1R^RyRRjRvR~RRiR((s2lib/python2.7/site-packages/dask/dataframe/core.pyRs !    "$g?tdefaultc `sj|}dt|}|dkrwttrOtdnttj|d|ddfStdj jd|}j }t fd |j D}g|D]}|j d f^q} t|d trvitj| |j d |jf|d f6} tj|| d |} t|j t|j f} t| ||| Sitj| df|d f6} tj|| d |} t| |||d jSd S( s Approximate row-wise and precise column-wise quantiles of DataFrame Parameters ---------- q : list/array of floats, default 0.5 (50%) Iterable of numbers ranging from 0 to 1 for the desired quantiles axis : {0, 1, 'index', 'columns'} (default 0) 0 or 'index' for row-wise, 1 or 'columns' for column-wise method : {'default', 'tdigest', 'dask'}, optional What method to use. By default will use dask's internal custom algorithm (``'dask'``). If set to ``'tdigest'`` will use tdigest for floats and ints and fallback to the ``'dask'`` otherwise. squantiles-concat--is+'q' must be scalar when axis=1 is specifiedRRjtf8tquantileR1c3`s%|]}t|VqdS(N(R(R tc(RtqRg(s2lib/python2.7/site-packages/dask/dataframe/core.pys siR_N(R6R/RKR}RRR$RRcRfRtRRDRbR^RRRSRRiR5R`RRR6RVRRk( RgRR1RtkeynameRjRvt quantilest_qtqnamesRRRk((RRRgs2lib/python2.7/site-packages/dask/dataframe/core.pyRs(   %"+c C`s|j}|jdkr?t|jdkr?tdn-|jdkrl|jdkrltdn|dkrddd g}n<tj|}tj |d}tj |}t |}|j d ||j d ||jd ||jd ||j|d ||jd |g}g|D]}|jdf^q0}d t||}itj|f|df6} tj|| d |} |jj} t| || dddgS(s.Currently, only numeric describe is supported iis)DataFrame contains only non-numeric data.iR#s,Cannot compute ``describe`` on object dtype.g?g?g?RRs describe--R_RkN(RtRCRURDRRwRRMRRR R}RrRsR|RRRRbR/R6tdescribe_aggregateR5R`RftdescribeR( RgRt percentilestpercentiles_methodRvtstatsRwt stats_namesRiRRRj((s2lib/python2.7/site-packages/dask/dataframe/core.pyRs. $  "cC`s|j|}|dkrUdj|j|}|j|d||} t|| Sdj|j|} t||d| d||} dj|j|} tt| |dtjgd| } t|}dj|j||}dj|j||}i}| j d f||d fs salign2-c`s.i|]$\}}t|df|fqS(i(R(R RYRn(R(s2lib/python2.7/site-packages/dask/dataframe/core.pys s ( t_emulateR$talignRR/RRoRzRaRRk( RgRRR1Rtmeta1tmeta2talignedRtdsk1tresult1Rtresult2((RRs2lib/python2.7/site-packages/dask/dataframe/core.pyRs      cC`s"|jtj||d|d|S(NRt overwrite(RR$R(RgRRRR((s2lib/python2.7/site-packages/dask/dataframe/core.pyRscC`s|jtj|S(N(RR$t combine_first(RgR((s2lib/python2.7/site-packages/dask/dataframe/core.pyRscC`s tdS(s7 bind operator method like DataFrame.add to this class N(R(RRiR((s2lib/python2.7/site-packages/dask/dataframe/core.pyt_bind_operator_methodscC`s)ddlm}|||d|d|S(Ni(t Resamplertclosedtlabel(ttseries.resampleR(RgtruleRRR((s2lib/python2.7/site-packages/dask/dataframe/core.pytresamplesc`sDjj|js(tdntjjj|}jd|}j j |}|j pyt |d }|j dkrj}nj|d |f}dt|fdt|D}tjj|fd||tdf||fs RR_(RvR RRRRttseriest frequenciest to_offsetRkRt_get_partitionst isAnchoredR|RR/RR6tboundary_sliceRbRRWR5R`R(Rgtoffsettdatetendt include_righttdivsRhR((RiRgs2lib/python2.7/site-packages/dask/dataframe/core.pyR s   c`s-jj|js(tdntjjj|}jd|}j j |}|dkrxj}n|fj|d}dt |fdt t |jD}tjj|f|dttdf|dfs RR_(RvR RRRRRRRRkRRR/RRRR6RRbRRWRR5R`R(RgRRtstartRRhR((RiRgs2lib/python2.7/site-packages/dask/dataframe/core.pytlasts    cC`sJddlm}t|gd|jd|jd|jd|ddd tS( s9Approximate number of unique rows. This method uses the HyperLogLog algorithm for cardinality estimation to compute the approximate number of unique rows. The approximate error is 0.406%. Parameters ---------- split_every : int, optional Group partitions into groups of this size while performing a tree-reduction. If set to False, no tree-reduction will be used. Default is 8. Returns ------- a float representing the approximate number of elements i(t hyperloglogRRRRtbiRj(RRRtcompute_hll_arrayt reduce_statetestimate_countR(RgRR((s2lib/python2.7/site-packages/dask/dataframe/core.pytnunique_approxs   cC`s|jtjS(s Return a dask.array of the values of this dataframe Warning: This creates a dask.array without precise shape information. Operations that depend on shape information, like slicing or reshaping, will not work. (RR6R7(Rg((s2lib/python2.7/site-packages/dask/dataframe/core.pyR72scC`se|jjdk odt| odtj|s=t|tod||jjkod|t|ddkS(s Test whether a key is an index level reference To be considered an index level reference, `key` must match the index name and must NOT match the name of any column (if a dataframe). RDN(( RRiRR1RMtisscalarRKRR(RgRn((s2lib/python2.7/site-packages/dask/dataframe/core.pyt_is_index_level_reference<s  c`s:t|tr)tfd|DSj|SdS(sb Test whether the input contains a reference to the index of the DataFrame/Series c3`s|]}j|VqdS(N(R(R R(Rg(s2lib/python2.7/site-packages/dask/dataframe/core.pys NsN(RKR}ReR(Rgtcolumns_or_index((Rgs2lib/python2.7/site-packages/dask/dataframe/core.pyt_contains_index_nameIsN(RRRRlRmRoRqRpRR;R0RRRRRRtRuRRRRRvRRRRRRRRRRRRRtsetterRRRRRR RRRRRRt __nonzero__RRRt__long__RR>RRt no_defaultRR RRWRRRR'R(R.R8R2R3R;R<RIRKRORQRRRRRRYR[RaRbRdReRRjRRRkRqRrRsRxR|RRRRRRRRRMtnanRfRRgRRRRRSRRRRRRR RRR7RR(((s2lib/python2.7/site-packages/dask/dataframe/core.pyRs(                        Qn   | ,+ ' /  -  /    0                 ) (             cC`sDt|tr@t|dr@|jtkr@td|ndS(sv Utility function to raise an error if an object column does not support a certain operation like `mean`. Rws%`%s` not supported with object seriesN(RKRSR|RwR#R(RR!((s2lib/python2.7/site-packages/dask/dataframe/core.pyRcSs-RScB`seZdZejZeeZdZ d@dZ e dZ e jdZ e dZe dZe dZedZed Zed Zd Ze d Zd ZdZd@eedZeejddZeejd@dddZdddZ ddZ!dZ"eejdd@dZ#eejdZ$e%ddZ&eejd@dZ'eejedZ(d@d d!Z)eejd@d"Z*eejd@d d#Z+eejd$d@d%Z,eejd$d@d&Z-eejd'Z.e/d(d)eejd@e0d*Z1eejd+Z2eeje3d,Z4eejd@d@d@d-Z5eejd.Z6eejd/Z7eejd0d@d@d1Z8eejd@d2Z9eejd3Z:eejd4Z;ed5Z<eejd@d6Z=eejd$d7Z>e%d8Z?e%d9Z@e/d(d)e3e0dAd:ZAeejd@ed;ZBeejd<d@ed=ZCeejd ed>ZDeeje3ed?ZERS(Bs Parallel Pandas Series Do not use this class directly. Instead use functions like ``dd.read_csv``, ``dd.read_parquet``, or ``dd.from_pandas``. Parameters ---------- dsk: dict The dask graph to compute this Series _name: str The key prefix that specifies which keys in the dask comprise this particular Series meta: pandas.Series An empty ``pandas.Series`` with names, dtypes, and index matching the expected output. divisions: tuple of index values Values along which we partition our blocks on the index See Also -------- dask.dataframe.DataFrame sseries-cC`st|trot|dkrot|ddtjr[|ddjdkr[d}qo|ddj}ntj |d|d|j S(NiiRRi(( RKRRURMRNRRRRRRSRi(RgRRR((s2lib/python2.7/site-packages/dask/dataframe/core.pyRxs !1 cC`s |jjS(N(RfRi(Rg((s2lib/python2.7/site-packages/dask/dataframe/core.pyRiscC`s7||j_t||}|j|_|j|_dS(N(RfRit _rename_daskRaRb(RgRitrenamed((s2lib/python2.7/site-packages/dask/dataframe/core.pyRis  cC`sdS(s Return dimensionality i((Rg((s2lib/python2.7/site-packages/dask/dataframe/core.pyRCscC`s |jfS(s  Return a tuple representing the dimensionality of a Series. The single element of the tuple is a Delayed result. Examples -------- >>> series.shape # doctest: +SKIP # (dd.Scalar,) (R(Rg((s2lib/python2.7/site-packages/dask/dataframe/core.pyRs cC`s |jjS(s Return data type (RfRw(Rg((s2lib/python2.7/site-packages/dask/dataframe/core.pyRwscC`s t|S(s Namespace of datetime methods (R7(Rg((s2lib/python2.7/site-packages/dask/dataframe/core.pytdtscC`s t|S(N(R9(Rg((s2lib/python2.7/site-packages/dask/dataframe/core.pytcatscC`s t|S(s Namespace for string methods (R8(Rg((s2lib/python2.7/site-packages/dask/dataframe/core.pyRscC`skttt|}|j|jx6ddgD](}t|j|s5|j|q5q5Wt|S(NRR( RxRyReRzR{R|RfRR}(RgR~taccessor((s2lib/python2.7/site-packages/dask/dataframe/core.pyRs c C`s(|jtjtjdddtdtS(s Number of bytes RtnbytesRjR(RR6RRMRRR(Rg((s2lib/python2.7/site-packages/dask/dataframe/core.pyRscC`st|j|jS(N(t_repr_data_seriesRfR(Rg((s2lib/python2.7/site-packages/dask/dataframe/core.pyRsc C`s|jd k r0djd|jd|j}ndjd|j}djd|jjd|jd|dt|jd t |j S( s have to overwrite footer sName: {name}, dtype: {dtype}RiRwsdtype: {dtype}sGDask {klass} Structure: {data} {footer} Dask Name: {name}, {task} tasksRRtfooterRN( RiRRdRwRRRR%RbRURa(RgR((s2lib/python2.7/site-packages/dask/dataframe/core.pyRs c C`s`ddlm}m}m}||sA||re|| re|rM|n |j}||_n|jtj|}|j r|rt |s||rt j t |jdd|j}|j|j} | jsd} t| nt| j|_q|j}n|r\|j|_|j|_|j|_|j|_|}n|S(sAlter Series index labels or name Function / dict values must be unique (1-to-1). Labels not contained in a dict / Series will be left as-is. Extra labels listed don't throw an error. Alternatively, change ``Series.name`` with a scalar value. Parameters ---------- index : scalar, hashable sequence, dict-like or callable, optional If dict-like or callable, the transformation is applied to the index. Scalar or hashable sequence-like will alter the ``Series.name`` attribute. inplace : boolean, default False Whether to return a new Series or modify this one inplace. sorted_index : bool, default False If true, the output ``Series`` will have known divisions inferred from the input series and the transformation. Ignored for non-callable/dict-like ``index`` or when the input series has unknown divisions. Note that this may only be set to ``True`` if you know that the transformed index is monotonicly increasing. Dask will check that transformed divisions are monotonic, but cannot check all the values between divisions, so incorrectly setting this can result in bugs. Returns ------- renamed : Series See Also -------- pandas.Series.rename i(t is_scalart is_list_liket is_dict_likeiRsGsorted_index=True, but the transformed index isn't monotonic_increasing(tpandas.api.typesRRRRRiRR$trenameRtcallableRRRSRRRkRtis_monotonic_increasingRRR$RRaRbRf( RgRtinplacet sorted_indexRRRtrestoldtnewR((s2lib/python2.7/site-packages/dask/dataframe/core.pyRs,#%         icC`sttj||S(N(RR$tround(Rgtdecimals((s2lib/python2.7/site-packages/dask/dataframe/core.pyRsRcC`s@ttj||||}ttj|jj|_|S(N(RR$t to_timestampRRRRRk(RgR*thowR1tdf((s2lib/python2.7/site-packages/dask/dataframe/core.pyRs!g?RcC`st||d|S(s Approximate quantiles of Series Parameters ---------- q : list/array of floats, default 0.5 (50%) Iterable of numbers ranging from 0 to 1 for the desired quantiles method : {'default', 'tdigest', 'dask'}, optional What method to use. By default will use dask's internal custom algorithm (``'dask'``). If set to ``'tdigest'`` will use tdigest for floats and ints and fallback to the ``'dask'`` otherwise. R(R(RgRR((s2lib/python2.7/site-packages/dask/dataframe/core.pyRs g?cC`s#ddlm}|||d|S(sA Approximate quantiles of Series used for repartitioning i(tpartition_quantilestupsample(tpartitionquantilesR(RgRRR((s2lib/python2.7/site-packages/dask/dataframe/core.pyt_repartition_quantiles-scC`st|tr|j|jkrdt||}ttj|||}tj||d||g}t|||j |jSt ddS(Nsindex-%sR_s[Series getitem in only supported for other series objects with matching partition structure( RKRSRkR/tpartitionwise_graphRRR5R`RfR(RgRnRiRhR((s2lib/python2.7/site-packages/dask/dataframe/core.pyt __getitem__3s!RecC`s|S(N((RgRR((s2lib/python2.7/site-packages/dask/dataframe/core.pyRt>scc`sOxHt|jD]7}|j|j}x|jD] }|Vq8WqWdS(N(RRRRt iteritems(RgRYRwtitem((s2lib/python2.7/site-packages/dask/dataframe/core.pyRBscC`sB|dkr$tdj|nidd6dd6j||S(NiRsNo axis named {0}(iRN(RRRdR(RR1((s2lib/python2.7/site-packages/dask/dataframe/core.pyR6Is cK`s#ddlm}||d||S(Ni(t SeriesGroupBytby(tdask.dataframe.groupbyR(RgRRR((s2lib/python2.7/site-packages/dask/dataframe/core.pytgroupbyPscC`stt|jd|S(NR(tsuperRSRr(RgR((s2lib/python2.7/site-packages/dask/dataframe/core.pyRrUsicC`s@t|dtjdtjd|jddd|d|jd|S( s Return Series of unique values in the object. Includes NA values. Returns ------- uniques : Series RRRjRR Rt series_nameR(RR6R RfRi(RgRR((s2lib/python2.7/site-packages/dask/dataframe/core.pyR YscC`s|jd|jS(NR(RRr(RgR((s2lib/python2.7/site-packages/dask/dataframe/core.pytnuniqueescC`sLt|dtjdtjdtjd|jjddd|d|d tS( NRRRRjRs value-countsRRR(RR$t value_countsR6tvalue_counts_aggregatetvalue_counts_combineRftsplit_out_on_index(RgRR((s2lib/python2.7/site-packages/dask/dataframe/core.pyRis    icC`s7t|dtjdtjd|jddd|d|S(NRRRjRsseries-nlargestRR(RR$tnlargestRf(RgRR((s2lib/python2.7/site-packages/dask/dataframe/core.pyR rscC`s7t|dtjdtjd|jddd|d|S(NRRRjRsseries-nsmallestRR(RR$t nsmallestRf(RgRR((s2lib/python2.7/site-packages/dask/dataframe/core.pyR xscC`stt|j|S(N(RRSR(RgR7((s2lib/python2.7/site-packages/dask/dataframe/core.pyR~sRi c`sttp1tp1to1t sRtdjtndt|fdt |j D}t j |d|g}|t krttj|ddt}n$t|dtt|dd}t|||jS(Ns4arg must be pandas.Series, dict or callable. Got {0}smap-c`s4i|]*\}}tj|f|fqS((R$tmap(R RYR!(targt na_actionRi(s2lib/python2.7/site-packages/dask/dataframe/core.pys s R_RtudfR(RKRRRFR1RcRdReR/RRoR5R`RRR$R RWR=RRRSRk(RgR RRjRhR((R RRis2lib/python2.7/site-packages/dask/dataframe/core.pyR s   $$cC`s|jtjS(N(RR$tdropna(Rg((s2lib/python2.7/site-packages/dask/dataframe/core.pyRscC`s"|jtjd|d|d|S(Ntlefttrightt inclusive(RR$tbetween(RgRRR((s2lib/python2.7/site-packages/dask/dataframe/core.pyRscC`s7|dk rtdn|jtjd|d|S(Ns'out' must be Nonetlowertupper(RRRR$tclip(RgRRR((s2lib/python2.7/site-packages/dask/dataframe/core.pyRs cC`s|jtjd|S(Nt threshold(RR$t clip_lower(RgR((s2lib/python2.7/site-packages/dask/dataframe/core.pyRscC`s|jtjd|S(NR(RR$t clip_upper(RgR((s2lib/python2.7/site-packages/dask/dataframe/core.pyRsRcC`s(tt|j|d|d|d|S(NRR1R(RRSR(RgRRR1R((s2lib/python2.7/site-packages/dask/dataframe/core.pyRs!cC`s|jtj||d|S(NR(RR$R(RgRRR((s2lib/python2.7/site-packages/dask/dataframe/core.pyRscC`s|S(N((Rg((s2lib/python2.7/site-packages/dask/dataframe/core.pytsqueezescC`s|jtj|S(N(RR$R(RgR((s2lib/python2.7/site-packages/dask/dataframe/core.pyRscC`sddlm}|||S(s! Create a Dask Bag from a Series i(tto_bag(RLR(RgRR((s2lib/python2.7/site-packages/dask/dataframe/core.pyRscC`s%|jtj|d|jj|S(NRj(RR$tto_frameRf(RgRi((s2lib/python2.7/site-packages/dask/dataframe/core.pyRscC`s|jjd|S(NR(RR(RgR((s2lib/python2.7/site-packages/dask/dataframe/core.pyRsc`s>dddfd}tj|_t|||dS(s4 bind operator method like Series.add to this class ic `sj|dk rtdn|j|}t||d|d|}t||d|d|d|S(Nslevel must be NoneR1RRj(RRR6RR(RgRtlevelRR1Rj(R(s2lib/python2.7/site-packages/dask/dataframe/core.pytmeths  N(RR)RR(RRiRR((Rs2lib/python2.7/site-packages/dask/dataframe/core.pyRsc`s>dddfd}tj|_t|||dS(s5 bind comparison method like Series.eq to this class ic`sx|dk rtdn|j|}|dkrLt||d|Std|}t|||d|SdS(Nslevel must be NoneR1R(RRR6RR(RgRRRR1R(t comparison(s2lib/python2.7/site-packages/dask/dataframe/core.pyRs  N(RR)RR(RRiR R((R s2lib/python2.7/site-packages/dask/dataframe/core.pyt_bind_comparison_methods c K`sq|tkrOttj|j|d|d|dt|}tjt|nt tj||||d||S(s& Parallel version of pandas.Series.apply Parameters ---------- func : function Function to apply convert_dtype : boolean, default True Try to find better dtype for elementwise function results. If False, leave as dtype=object. $META args : tuple Positional arguments to pass to function in addition to the value. Additional keyword arguments will be passed as keywords to the function. Returns ------- applied : Series or DataFrame if func returns a Series. Examples -------- >>> import dask.dataframe as dd >>> s = pd.Series(range(5), name='x') >>> ds = dd.from_pandas(s, npartitions=2) Apply a function elementwise across the Series, passing in extra arguments in ``args`` and ``kwargs``: >>> def myadd(x, a, b=1): ... return x + a + b >>> res = ds.apply(myadd, args=(2,), b=1.5) By default, dask tries to infer the output metadata by running your provided function on some fake data. This works well in many cases, but can sometimes be expensive, or even fail. To avoid this, you can manually specify the output metadata with the ``meta`` keyword. This can be specified in many forms, for more information see ``dask.dataframe.utils.make_meta``. Here we specify the output is a Series with name ``'x'``, and dtype ``float64``: >>> res = ds.apply(myadd, args=(2,), b=1.5, meta=('x', 'f8')) In the case where the metadata doesn't change, you can also pass in the object itself directly: >>> res = ds.apply(lambda x: x + 1, meta=ds) See Also -------- dask.Series.map_partitions t convert_dtypeRXRRj( RRR$RRvRWR+R,t meta_warningR(RgRR"RjRXtkwds((s2lib/python2.7/site-packages/dask/dataframe/core.pyRs7 cC`s_ddlm}t|ts.tdn|||gdd}t||dtd|S(Ni(RVs%other must be a dask.dataframe.SeriesR1RlR(RRVRKRSRctcov_corrRW(RgRRURRVR((s2lib/python2.7/site-packages/dask/dataframe/core.pytcov. s tpearsonc C`sddlm}t|ts.tdn|dkrItdn|||gdd}t||dtdtd |S( Ni(RVs%other must be a dask.dataframe.SeriesR's-Only Pearson correlation has been implementedR1tcorrRlR(RRVRKRSRcRR%RW(RgRRRURRVR((s2lib/python2.7/site-packages/dask/dataframe/core.pyR(6 s cC`sLt|tstdn|j|dkr6|n |j|d|S(Nslag must be an integeriR(RKRRcR(R[(RgtlagR((s2lib/python2.7/site-packages/dask/dataframe/core.pytautocorrC s'cC`s4|jtjd|d|}tt|jS(NRtdeep(RR$t memory_usageR2RR(RgRR+R((s2lib/python2.7/site-packages/dask/dataframe/core.pyR,J sN((FRRRRRRSt_partition_typeRRFRR_RRRRiRRCRRwR RRRRRRRRRR RRRRRRRtRRR6RRrR RRR R RR>RR RRWRRRRRRRRRRRRR!RR&R(R*R,(((s2lib/python2.7/site-packages/dask/dataframe/core.pyRS\s      =                    ?    RcB`saeZejZeeZdZddddddddd d d d d ddhZ ddddddddddddddhZ dZ dZ e d Zd-d!Zd"ed#Zeejed$Zeejed%Zed&Zeejd'd-d(Zeejd)Zeejd*d+ged-d,ZRS(.sindex-t nanosecondt microsecondt millisecondt dayofyeartminutethourtdayt dayofweektsecondtweektweekdayt weekofyeartmonthtquartertyeartknowntas_knownt as_unknowntadd_categoriesRtremove_categoriestreorder_categoriest as_orderedtcodestremove_unused_categoriestset_categoriest as_unorderedtorderedtrename_categoriescC`sat|jr.||jkr.t|j|S||jkrMt|j|Std|dS(Ns"'Index' object has no attribute %r(RARwt_cat_attributesRRt_dt_attributesRtAttributeError(RgRn((s2lib/python2.7/site-packages/dask/dataframe/core.pyt __getattr__` s cC`sKtt|j}|j|jt|jrG|j|jn|S(N(RRRtextendRKRARwRJ(RgR((s2lib/python2.7/site-packages/dask/dataframe/core.pyRg s cC`s%d}t|j|jjdS(Ns%'{0}' object has no attribute 'index'(RLRdRR(RgR((s2lib/python2.7/site-packages/dask/dataframe/core.pyRn scC`stj|d|jS(NRi(RRRRi(RgRR((s2lib/python2.7/site-packages/dask/dataframe/core.pyRs sicC`sd||jf}itj|jdftd|f|df6}tj||d|g}t|||j|jd }|r|j }n|S(s\ First n items of the Index. Caveat, this only checks the first partition. s head-%d-%siR_i( RbRRR R5R`RRfRkR(RgRRRiRhRR((s2lib/python2.7/site-packages/dask/dataframe/core.pyRv s1cC`s2|jtjd|jjd|jdd|S(NRjRRR(RR$RRvR_(RgR((s2lib/python2.7/site-packages/dask/dataframe/core.pyR s cC`s2|jtjd|jjd|jdd|S(NRjRRR(RR$RRvR_(RgR((s2lib/python2.7/site-packages/dask/dataframe/core.pyR s c C`s(|jtjtjdddtd|S(NRs index-countRjR(RR6t index_countRMRR(RgR((s2lib/python2.7/site-packages/dask/dataframe/core.pyRr s ic C`st|jtjrl|dk r0tdn|jj|}|jt j|d|dddt }nE|jj|d|}|jt j|ddd|d|dt }|dkr|j }nt ||d|S(Ns*PeriodIndex doesn't accept `freq` argumentRjRR[R\R*( RKRfRRt PeriodIndexRRRvR[RR$RR*R](RgRZR*RjR((s2lib/python2.7/site-packages/dask/dataframe/core.pyR[ s      cC`s|jtjd|jjS(NRj(RR$t to_seriesRf(Rg((s2lib/python2.7/site-packages/dask/dataframe/core.pyRQ stua_argsRcC`s|stntdkrI|jtj||d|jj||S|dk rmtdjtn|jtjd|jjSdS(Ns0.24.0RjsNThe 'name' keyword was added in pandas 0.24.0. Your version of pandas is '{}'.( RRCRR$RRfRRRd(RgRRi((s2lib/python2.7/site-packages/dask/dataframe/core.pyR s    N(RRRRRR-RRGRR_RKRJRMRRRRRRWRR RRRRrR[RQR(((s2lib/python2.7/site-packages/dask/dataframe/core.pyRP s4               Rc B`s'eZdZejZeeZdZ dPdZ e dZ e jdZ e dZdZdZdZd Zd Zd Zd Ze d Ze dZe dZeejdZeejdZeejdPdPdZeedPdPedZ eejddPdPdZ!eejddPdPdZ"eejdPdZ#e$e%dPdPdPdZ%eejdZ&eejddgdPdPdZ'dZ(eejdPdZ)eejddPdPd Z*eejdPdPdPd!Z+eejd"Z,eejd#Z-eejdPd$Z.eejdPd%d&d'Z/ed(Z0d)Z1eejdd*Z2ddPd+Z3e4d&d,Z5eejd&d-d.Z6d/dPdPdPeedQedPdPd2 Z7eejdPd3d4d4dPdPd5Z8eejed6Z9eejd7Z:eejed8d9Z;e4d:Z<e4d;Z=e>d<d=d&dPedPdRe?d>Z@eejd?d@ZAeejd&dAZBeejdPedBZCeejdCdPedDZDdPeedEZEeejeedFZFdPdPdPdGdHZGedIZHeejddJZIdKZJdLZKdMZLdNZMdOZNRS(Ssc Parallel Pandas DataFrame Do not use this class directly. Instead use functions like ``dd.read_csv``, ``dd.read_parquet``, or ``dd.from_pandas``. Parameters ---------- dsk: dict The dask graph to compute this DataFrame name: str The key prefix that specifies which keys in the dask comprise this particular DataFrame meta: pandas.DataFrame An empty ``pandas.DataFrame`` with names, dtypes, and index matching the expected output. divisions: tuple of index values Values along which we partition our blocks on the index s dataframe-cC`st|trot|dkrot|ddtjr[|ddjdkr[d}qo|ddj}ntj |d|d|j S(NiiRRD(( RKRRURMRNRRRRRRRD(RgRRR((s2lib/python2.7/site-packages/dask/dataframe/core.pyR s !1 cC`s |jjS(N(RfRD(Rg((s2lib/python2.7/site-packages/dask/dataframe/core.pyRD scC`s7t||}|j|_|j|_|j|_dS(N(RRfRbRa(RgRDR((s2lib/python2.7/site-packages/dask/dataframe/core.pyRD s  cC`sddlm}||S(sLPurely integer-location based indexing for selection by position. Only indexing the column positions is supported. Trying to select row positions will raise a ValueError. See :ref:`dataframe.indexing` for more. Examples -------- >>> df.iloc[:, [2, 0, 1]] # doctest: +SKIP i(t _iLocIndexer(RRS(RgRS((s2lib/python2.7/site-packages/dask/dataframe/core.pytiloc s c C`sdt||}tj|s7t|ttfrt|jjtj tj frx||jj krx|j |Sn|jt |}ttj|||}tj||d|g}t||||jSt|trPddlm}td|j|j|jfD}|rB||jj rB|j|qP|j |Snt|tjtfst | rt!|st"|r|jt |}ttj|||}tj||d|g}t||||jSt|t#r|j|jkr5ddl$m%}|||g\}}nttj|||}tj||d||g}t||||jSt&|dS(Ns getitem-%sR_i(tis_float_dtypecs`s|]}t|tVqdS(N(RKR(R RY((s2lib/python2.7/site-packages/dask/dataframe/core.pys si(t_maybe_align_partitions('R/RMRRKRRRfRRRt DatetimeIndexRPRDRt _extract_metaRRRR5R`RRkR RRUReRtsteptstopRwRTRNR}R1RFRGRSRRVR( RgRnRiRjRhRRUtis_integer_sliceRV((s2lib/python2.7/site-packages/dask/dataframe/core.pyR s<$! %c`st|ttfrRttrR|jfdt|jD}nft|tjrtt rt|}|jfd|D}n|ji|6}|j |_ |j |_ |j |_ |j |_ dS(Nc`s#i|]\}}||qS(((R R!R(R(s2lib/python2.7/site-packages/dask/dataframe/core.pys ) s c`si|]}|qS(((R R!(R(s2lib/python2.7/site-packages/dask/dataframe/core.pys . s ( RKRR}RtassigntzipRDRRRRaRbRfRk(RgRnRR((Rs2lib/python2.7/site-packages/dask/dataframe/core.pyt __setitem__' s$" "   cC`s@|j|gdd}|j|_|j|_|j|_dS(NR1i(RRaRbRf(RgRnR((s2lib/python2.7/site-packages/dask/dataframe/core.pyt __delitem__7 s  cC`scytj|dj}Wntk r2d}nX||krL|||Q s(RxRyReRzR{RDR}(RgR~((s2lib/python2.7/site-packages/dask/dataframe/core.pyRN scC`s |jjS(N(RDR$(Rg((s2lib/python2.7/site-packages/dask/dataframe/core.pyt_ipython_key_completions_V scC`sdS(s Return dimensionality i((Rg((s2lib/python2.7/site-packages/dask/dataframe/core.pyRCY scC`s2t|j}tt|j|}||fS(sB Return a tuple representing the dimensionality of the DataFrame. The number of rows is a Delayed result. The number of columns is a concrete integer. Examples -------- >>> df.size # doctest: +SKIP (Delayed('int-07f06075-5ecc-4d77-817e-63c69a9188a8'), 2) (RURDR2RR(Rgtcol_sizetrow_size((s2lib/python2.7/site-packages/dask/dataframe/core.pyR^ s cC`s |jjS(s Return data types (Rftdtypes(Rg((s2lib/python2.7/site-packages/dask/dataframe/core.pyRgo scC`s |jjS(N(Rftget_dtype_counts(Rg((s2lib/python2.7/site-packages/dask/dataframe/core.pyRht scC`s |jjS(N(Rftget_ftype_counts(Rg((s2lib/python2.7/site-packages/dask/dataframe/core.pyRix scC`s,|jjd|d|j}|t|S(Ntincludetexclude(Rft select_dtypesRDR}(RgRjRktcs((s2lib/python2.7/site-packages/dask/dataframe/core.pyRl| sc K`s|rtdn|}~|dk r7t|n|riddlm} | ||d|d||Sddlm} | ||d|d|d||SdS( sSet the DataFrame index (row labels) using an existing column This realigns the dataset to be sorted by a new column. This can have a significant impact on performance, because joins, groupbys, lookups, etc. are all much faster on that column. However, this performance increase comes with a cost, sorting a parallel dataset requires expensive shuffles. Often we ``set_index`` once directly after data ingest and filtering and then perform many cheap computations off of the sorted dataset. This function operates exactly like ``pandas.set_index`` except with different performance costs (it is much more expensive). Under normal operation this function does an initial pass over the index column to compute approximate qunatiles to serve as future divisions. It then passes over the data a second time, splitting up each input partition into several pieces and sharing those pieces to all of the output partitions now in sorted order. In some cases we can alleviate those costs, for example if your dataset is sorted already then we can avoid making many small pieces or if you know good values to split the new index column then we can avoid the initial pass over the data. For example if your new index is a datetime index and your data is already sorted by day then this entire operation can be done for free. You can control these options with the following parameters. Parameters ---------- df: Dask DataFrame index: string or Dask Series npartitions: int, None, or 'auto' The ideal number of output partitions. If None use the same as the input. If 'auto' then decide by memory use. shuffle: string, optional Either ``'disk'`` for single-node operation or ``'tasks'`` for distributed operation. Will be inferred by your current scheduler. sorted: bool, optional If the index column is already sorted in increasing order. Defaults to False divisions: list, optional Known values on which to separate index values of the partitions. See https://docs.dask.org/en/latest/dataframe-design.html#partitions Defaults to computing this with a single pass over the data. Note that if ``sorted=True``, specified divisions are assumed to match the existing partitions in the data. If this is untrue, you should leave divisions empty and call ``repartition`` after ``set_index``. inplace : bool, optional Modifying the DataFrame in place is not supported by Dask. Defaults to False. compute: bool Whether or not to trigger an immediate computation. Defaults to False. Examples -------- >>> df2 = df.set_index('x') # doctest: +SKIP >>> df2 = df.set_index(d.x) # doctest: +SKIP >>> df2 = df.set_index(d.timestamp, sorted=True) # doctest: +SKIP A common case is when we have a datetime column that we know to be sorted and is cleanly divided by day. We can set this index for free by specifying both that the column is pre-sorted and the particular divisions along which is is separated >>> import pandas as pd >>> divisions = pd.date_range('2000', '2010', freq='1D') >>> df2 = df.set_index('timestamp', sorted=True, divisions=divisions) # doctest: +SKIP s%The inplace= keyword is not supportedi(tset_sorted_indexRRk(t set_indexRN(RRtcheck_divisionstshuffleRnRo( RgRRtsortedRRkRRt pre_sortedRnRo((s2lib/python2.7/site-packages/dask/dataframe/core.pyRo sC  icC`sCd}t|dtjdtjd|jd|d|d|d|S( Nsdataframe-nlargestRRRjRRRRD(RR$R Rf(RgRRDRR((s2lib/python2.7/site-packages/dask/dataframe/core.pyR  scC`sCd}t|dtjdtjd|jd|d|d|d|S( Nsdataframe-nsmallestRRRjRRRRD(RR$R Rf(RgRRDRR((s2lib/python2.7/site-packages/dask/dataframe/core.pyR  scK`s#ddlm}||d||S(Ni(tDataFrameGroupByR(RRt(RgRRRt((s2lib/python2.7/site-packages/dask/dataframe/core.pyR sc K`st|d|d|d||S(NRDRR(R:(RgRDRRR((s2lib/python2.7/site-packages/dask/dataframe/core.pyR: scK`sx|jD]\}}t|tp^t|p^t|p^tjjj|p^t |st dj t |j nt|r ||||>> import dask.dataframe as dd >>> df = pd.DataFrame({'x': [1, 2, 3, 4, 5], ... 'y': [1., 2., 3., 4., 5.]}) >>> ddf = dd.from_pandas(df, npartitions=2) Apply a function to row-wise passing in extra arguments in ``args`` and ``kwargs``: >>> def myadd(row, a, b=1): ... return row.sum() + a + b >>> res = ddf.apply(myadd, axis=1, args=(2,), b=1.5) By default, dask tries to infer the output metadata by running your provided function on some fake data. This works well in many cases, but can sometimes be expensive, or even fail. To avoid this, you can manually specify the output metadata with the ``meta`` keyword. This can be specified in many forms, for more information see ``dask.dataframe.utils.make_meta``. Here we specify the output is a Series with name ``'x'``, and dtype ``float64``: >>> res = ddf.apply(myadd, axis=1, args=(2,), b=1.5, meta=('x', 'f8')) In the case where the metadata doesn't change, you can also pass in the object itself directly: >>> res = ddf.apply(lambda row: row + 1, axis=1, meta=ddf) See Also -------- dask.DataFrame.map_partitions R1t broadcasttrawtreduces0.23.0t result_typeisEdd.DataFrame.apply only supports axis=1 Try: df.apply(func, axis=1)RXRRjN(R6RRCt setdefaultRzRRRR$RRvRWR+R,R#R( RgRR1RRRRXRjR$t pandas_kwargsR((s2lib/python2.7/site-packages/dask/dataframe/core.pyR' s"?     RIcC`sttj||d|S(NRj(RR$tapplymap(RgRRj((s2lib/python2.7/site-packages/dask/dataframe/core.pyR scC`sttj||S(N(RR$R(RgR((s2lib/python2.7/site-packages/dask/dataframe/core.pyR scC`st||d|S(NR(R%(RgRUR((s2lib/python2.7/site-packages/dask/dataframe/core.pyR& sR'cC`s1|dkrtdnt||td|S(NR's-Only Pearson correlation has been implementedR(RR%RW(RgRRUR((s2lib/python2.7/site-packages/dask/dataframe/core.pyR( s cC`s|dkr$ddl}|j}ntt|g}t|jdkr|jd|jdt|jt ||dSi}|r|j i|j d6|j d6n|r|j i|j tjdtd6ntt|jtj|j}|r|d}|d}|jt||jdjt|jdd lm} tg|jD]} t| | ^q{d } d | } gt|j||jD]-} | j| | d| d | d ^q}nt|jddg}|j|gt|jjj dtD]} d| ^q@}|jdjdj!||r|dj"}|jdjt#|nt ||dS(s6 Concise summary of a Dask DataFrame. iNsIndex: 0 entriessEmpty %sRRrR,s Data columns (total {} columns):(t pprint_thingis{!s:<%d} {} non-null {}iiRitColumnsRns%s(%d)s dtypes: {}s, smemory usage: {} ($RtsyststdoutRReRURDRRR#RzRRrRR$R,RWRR]RRRORR7RDRdtpandas.io.formats.printingRRRgRNRrRRRRR"(RgtbuftverboseR,Rtlinest computationsRtcountsRR!tspacetcolumn_templateRt column_infot dtype_countst memory_int((s2lib/python2.7/site-packages/dask/dataframe/core.pytinfo sB     ')*  2 L 8cC`s:|jtjd|d|}|j|jj}|S(NRR+(RR$R,RRR(RgRR+R((s2lib/python2.7/site-packages/dask/dataframe/core.pyR, sRsc C`s2ddlm}||d|d|d|d|S(s/ Create a spreadsheet-style pivot table as a DataFrame. Target ``columns`` must have category dtype to infer result's ``columns``. ``index``, ``columns``, ``values`` and ``aggfunc`` must be all scalar. Parameters ---------- values : scalar column to aggregate index : scalar column to be index columns : scalar column to be columns aggfunc : {'mean', 'sum', 'count'}, default 'mean' Returns ------- table : DataFrame i(t pivot_tableRRDR7taggfunc(treshapeR(RgRRDR7RR((s2lib/python2.7/site-packages/dask/dataframe/core.pyR scC`sddlm}||S(Ni(t to_records(RLR(RgRR((s2lib/python2.7/site-packages/dask/dataframe/core.pyR scC`sO|jjd|dt}|jjd|dt|jdt|jS(NRRRRiR( Rtto_htmlRt _HTML_FMTRdR%RbRURa(RgRR((s2lib/python2.7/site-packages/dask/dataframe/core.pyR s !cC`sV|j}|j}g|jD]\}}t|d|^q}tj|ddS(NRR1i(RfRRRRRRV(RgRjRR&Rwt series_list((s2lib/python2.7/site-packages/dask/dataframe/core.pyR s  1sg
Dask DataFrame Structure:
{data}
Dask Name: {name}, {task} tasks
cC`sU|jjdddtdt}|jjd|dt|jdt|j S(NRiRtnotebookRRiR( RRRRWRRdR%RbRURa(RgR((s2lib/python2.7/site-packages/dask/dataframe/core.pyt _repr_html_ s  !cC`s{t|tr|n|g}g|D]}|j|r%|^q%}||}|j|rw|jd|j}n|S(s~ Parameters ---------- columns_or_index Column or index name, or a list of these Returns ------- dd.DataFrame Dask DataFrame with columns corresponding to each column or index level in columns_or_index. If included, the column corresponding to the index level is named _index t_index(RKR}t_is_column_label_referenceRR\R(RgRRt column_namest selected_df((s2lib/python2.7/site-packages/dask/dataframe/core.pyt_select_columns_or_index s ( cC`s8t| o7tj|s+t|to7||jkS(s Test whether a key is a column label reference To be considered a column label reference, `key` must match the name of at least one column. (R1RMRRKRRD(RgRn((s2lib/python2.7/site-packages/dask/dataframe/core.pyR s N(RR((ORRRRRRR-RRERR_RRRRDRRTRR^R_RaRMRRdRCRRgR RhRiRlRWRRoR R RRR:R\RRzR}RRRRRRRRRRtRR6RR RRRRRR!R>RRRRR&R(RR,RRRRRRRR(((s2lib/python2.7/site-packages/dask/dataframe/core.pyR s    +       S                U    !  V    1     RtsubRtdivttruedivtfloordivtmodtpowtraddtrsubtrmultrdivtrtruedivt rfloordivtrmodtrpowtlttgttletgetneteqc`sAtto@jdko@jo@tfd|DS(sP This Series is broadcastable against another dataframe in the sequence ic3`sE|];}t|trjt|jt|jfkVqdS(N(RKRRkRRDR(R R(Rw(s2lib/python2.7/site-packages/dask/dataframe/core.pys L s(RKRSRRRe(tdfsRw((Rws2lib/python2.7/site-packages/dask/dataframe/core.pytis_broadcastableE s  c `s|jdt}|jdd}|jdt}t|dt|||}t|}ddlm}||}g|D]$}t |t t t fr|^q} g| D]} t | t r| ^q} xt | D]\} t t sqntfd| Ds@dt|} t| njdkrjd t jdD| | >> elemwise(operator.add, df.x, df.y) # doctest: +SKIP RjRR\Ri(RVc3`s5|]+}j p,tjd|jkVqdS(iN(RHRUR(R R(RJ(s2lib/python2.7/site-packages/dask/dataframe/core.pys y ss[When combining dask arrays with dataframes they must match chunking exactly. Operation: %scS`s#i|]\}}||dqS(i((R RYtd((s2lib/python2.7/site-packages/dask/dataframe/core.pys  s iR_ics`s|]}t|dVqdS(RN(R|(R R((s2lib/python2.7/site-packages/dask/dataframe/core.pys ss>elemwise with 2 or more DataFrames and Scalar is not supportedtfunctionRN()tpopRRRWR!R/t_maybe_from_pandasRRVRKRR^R*RRdRRCtrechunkRRkRRURRR$RTRHRRRR}RRR5R`RRfRvR?RRR`(RRXRRjRR\RbRVR tdasksRRRYRRkt_is_broadcastableRRhRRR:R((RJs2lib/python2.7/site-packages/dask/dataframe/core.pyRP s^   1( & +8   * )=!cC`st|trXt|dkr.|d}qXt|dkrOtdqXd}n|dk rt|t|krtdtt|tt|fnt|trt|j t|j krt dtt|j tt|j fqnt|t tt frq|j |_ |j|_|j|_t|t s|j|_qnA|dk rdt|jt|jf}t|n|SdS(s Handle out parameters If out is a dask.DataFrame, dask.Series or dask.Scalar then this overwrites the contents of it with the result iis(The out parameter is not fully supportedsDMismatched types between result and out parameter. out=%s, result=%ssLMismatched columns count between result and out parameter. out=%s, result=%ssHThe out parameter is not fully supported. Received type %s, expected %s N(RKRRURRReRcRRRDRRSR^RfRbRaRkR(RRR((s2lib/python2.7/site-packages/dask/dataframe/core.pyR` s4  $+4    cC`saddlm}g|D]@}t|s5t|rQt| rQ||dn|^q}|S(Ni(t from_pandas(RLRRFRER1(RRR((s2lib/python2.7/site-packages/dask/dataframe/core.pyR sJc`su|r||pintdttrNjn|;fdt|DS(NRc`s&i|]}j|k|qS((RT(R RY(Rth(s2lib/python2.7/site-packages/dask/dataframe/core.pys s (R RRFt_valuesR(RtnpartsRR((RRs2lib/python2.7/site-packages/dask/dataframe/core.pyt hash_shard s   c`sEtjdt|djtfdt|DS(s, Split dataframe into k roughly equal parts iic`s/i|]%}j||d!|qS(i(RT(R RY(RRk(s2lib/python2.7/site-packages/dask/dataframe/core.pys s (RMtlinspaceRURRR(RR!((RRks2lib/python2.7/site-packages/dask/dataframe/core.pyt split_evenly s(cC`s=|j}t|tjr9tjgd|j}n|S(NR(RRKRRt MultiIndexRR(RR((s2lib/python2.7/site-packages/dask/dataframe/core.pyR  s cC`s||S(N((RR((s2lib/python2.7/site-packages/dask/dataframe/core.pyR sc `stdkrtn|dkr0t}nj| |j| |dkrz|rktdn|}|}n%|dkrt}n|j| tttfsgngD]}t|tr|^q}td|D}t |dkrtdn|j }| dkr@d} n@| t krU|} n+| dksqt| t  rtdnt |p|f|||| | | | }dj|pt|t dkr)td tr) r)fd td jD}n%fd t|D}| r| dkrd |}d |}x{t|D]m}td |d f| | | f|||f block Function to operate on each block of data aggregate : function concatenated-block -> block Function to operate on the concatenated result of chunk combine : function concatenated-block -> block, optional Function to operate on intermediate concatenated results of chunk in a tree-reduction. If not provided, defaults to aggregate. $META token : str, optional The name to use for the output keys. chunk_kwargs : dict, optional Keywords for the chunk function only. aggregate_kwargs : dict, optional Keywords for the aggregate function only. combine_kwargs : dict, optional Keywords for the combine function only. split_every : int, optional Group partitions into groups of this size while performing a tree-reduction. If set to False, no tree-reduction will be used, and all intermediates will be concatenated and passed to ``aggregate``. Default is 8. split_out : int, optional Number of output partitions. Split occurs after first chunk reduction. split_out_setup : callable, optional If provided, this function is called on each chunk before performing the hash-split. It should return a pandas object, where each row (excluding the index) is hashed. If not provided, the chunk is hashed as is. split_out_setup_kwargs : dict, optional Keywords for the `split_out_setup` function only. kwargs : All remaining keywords will be passed to ``chunk``, ``aggregate``, and ``combine``. Examples -------- >>> def chunk(a_block, b_block): ... pass >>> def agg(df): ... pass >>> apply_concat_apply([a, b], chunk=chunk, aggregate=agg) # doctest: +SKIP s+`combine_kwargs` provided with no `combine`cs`s|]}|jVqdS(N(R(R R ((s2lib/python2.7/site-packages/dask/dataframe/core.pys Gsis1All arguments must have same number of partitionsiis#split_every must be an integer >= 2s {0}-chunk-{1}ic`s1i|]'\}}|fd|dfqS(i((R RYRn(RJR(s2lib/python2.7/site-packages/dask/dataframe/core.pys [s c `sbi|]X}tgD]*}t|tr=|j|fn|^qfd|dfqS(i(RRKRRb(R RYR(RJRXRR(s2lib/python2.7/site-packages/dask/dataframe/core.pys ^s ssplit-%ssshard-%ss{0}-combine-{1}s {0}-agg-{1}RRR_N( RRRzRRKRR}RRxRURRRR/RdR!RRoRRRRR[RRRRWR=RR5R`R( RXRRRRjRRRRRRRRRR RRt token_keyRht split_prefixt shard_prefixRYRRR!tdepthtpart_itindstconct meta_chunkRRk((RJRXRRs2lib/python2.7/site-packages/dask/dataframe/core.pytapply_concat_apply s9            (        ,  - (+,( 1"   +cC`st|ttfr)|r"|jS|jSt|trXg|D]}t||^q?St|trtg|D]}t||^qqSt|tri}x%|D]}t|||||Cannot infer dataframe metadata with a `dask.delayed` argumentN( RKR^RRvRfR}RXRRR3R(RRuRRR!((s2lib/python2.7/site-packages/dask/dataframe/core.pyRXs & c O`sNtt|d|jdt$|t|tt|tSWdQXdS(s Apply a function using args / kwargs. If arguments contain dd.DataFrame / dd.Series, using internal cache (``_meta``) for calculation RN(R?R!RRRXRW(RRXR((s2lib/python2.7/site-packages/dask/dataframe/core.pyRs%c O`s1|jdt}|jdd}|jdt}t|sHt|dk rit|||}n!t|}t||||}dj||}ddl m }t |}||}g|D]}t |t r|^q} | rtt| dddnd} |tkr;t|d t||}nt|d| }td |Drit|tg|D]} | jdf^qvf|f|df6} tj|| d |} t| ||St|pt|stt|gd| }nt|}g}g}x|D]} t | t rV|j| |j| q!nt| } t| \}}|r|j||j|q!|j| q!Wi}xO|jD]A\}}t|}t|\}}|j||||sR_t_funcRfN(+RRRRWRtAssertionErrorR/R!RdRRVRRKRRR=RRdRRRbR5R`R^RQR'R[RR+R4RNRRtapply_and_enforceRkRRURRR$RTRHRR(RRXRRjRiR\RRVRRt meta_indexR RRRZR_targ2t collectionstkwargs3R!R~RhRkRJ((s2lib/python2.7/site-packages/dask/dataframe/core.pyRs    (( >          +8   cO`s|jd}|jd}|||}t|sQt|sQt|rt|sa|St|rtjtj|jtj|jst dq|j}n |j }t ||S|S(ssApply a function, and enforce the output to match meta Ensures the output has the same columns, even if empty.RRfsRThe columns in the computed data do not match the columns in the provided metadata( RRERFRGRURMt array_equalt nan_to_numRDRRit_rename(RXRRRjRR((s2lib/python2.7/site-packages/dask/dataframe/core.pyR!s$     cC`sWt|t st|tkr&|St|trDt|}nt|rt|rh|j}nt|tj stj |}nt |t |jkrt |t |jkr|j |jr|S|j dt}||_|St|st|rSt|s't|r3|j}n|j|krF|S|j|S|S(su Rename columns of pd.DataFrame or name of pd.Series. Not for dd.DataFrame or dd.Series. Parameters ---------- columns : tuple, string, pd.DataFrame or pd.Series Column names, Series name or pandas instance which has the target column names / name. df : pd.DataFrame or pd.Series target DataFrame / Series to be renamed R+(RKRRRRR}RERDRRRRURetequalsRRRFRGRiR(RDR((s2lib/python2.7/site-packages/dask/dataframe/core.pyR9s0        cC`st|tstt||j}djt||}tt|||}tj ||d|g}t ||||j S(s Destructively rename columns of dd.DataFrame or name of dd.Series. Not for pd.DataFrame or pd.Series. Internaly used to overwrite dd.DataFrame.columns and dd.Series.name We can't use map_partition because it applies function then rename Parameters ---------- df : dd.DataFrame or dd.Series target DataFrame / Series to be renamed names : tuple, string Column names/Series name s rename-{0}R_( RKRRRRfRdR/RR5R`RRk(RtnamestmetadataRiRhR((s2lib/python2.7/site-packages/dask/dataframe/core.pyRfs Rc`smtj}|jdkr7|jdd|nt|tsLtdddg}||krvtdn|dkrd}n|}t|trt j|j j }n|j j }t |r|j fd}t}nd }t}gtjd t|} tdkrd | t jgd t} tit jgd |j d| df6|jddgStjtjg} |j}|dkrtj|jtjstj|jtjrddlm} | ddddlmm } d| fdt!|j"D}d| }i|| t#|f|df6}nddlm$m%}d| fdt!|j"D}d| }i||g|j&t#|f|df6}t'||}t(j)||d|g}||||| S(sApproximate quantiles of Series. Parameters ---------- q : list/array of floats Iterable of numbers ranging from 0 to 100 for the desired quantiles method : {'default', 'tdigest', 'dask'}, optional What method to use. By default will use dask's internal custom algorithm (``'dask'``). If set to ``'tdigest'`` will use tdigest for floats and ints and fallback to the ``'dask'`` otherwise. itkindt mergesortRRattdigests1method can only be 'default', 'dask' or 'tdigest'c`stj|dfS(N(RRRSR(ttsk(tdf_nameR(s2lib/python2.7/site-packages/dask/dataframe/core.pyRRcS`s t|dfS(Ni(R(R((s2lib/python2.7/site-packages/dask/dataframe/core.pyRRids quantiles-RwRiR(timport_requiredtcricks=crick is a required dependency for using the t-digest method.(t_tdigest_chunkt_percentiles_from_tdigestsquantiles_tdigest-1-c`s4i|]*\}}t|dff|fqS(R7(R(R RYRn(RRi(s2lib/python2.7/site-packages/dask/dataframe/core.pys s squantiles_tdigest-2-(t _percentiletmerge_percentiless quantiles-1-c`s7i|]-\}}t|dff|fqS(R7(R(R RYRn(RRitqs(s2lib/python2.7/site-packages/dask/dataframe/core.pys s s quantiles-2-R_N(*RMRRCtsortRKRSRRRRRRvRRFRiR^RR/RURRfRRRRt issubdtypeRwtfloatingtintegert dask.utilsRtdask.array.percentileRRRRoRrRRRR R5R`(RRRt q_ndarraytallowed_methodstinternal_methodRjt finalize_tskRRt empty_indext new_divisionsRRtval_dskRt merge_dskRRhR((RRRRiRRs2lib/python2.7/site-packages/dask/dataframe/core.pyRsj          +  0       c`s|dkrd}n|dkr0tdn|tkrH|j}n+|dksdt|t rstdn|j}|rt|jdkrtdnt ||||}rdnd}dj ||j fdt |j D}d j ||j }|j} } d } x| |kr|t| } x[t t|t| D]>\} } tg| D]}|f^qf|| | f= 2s#split_every must be an integer >= 2s(scalar only valid for 2 column dataframeR(R&s {0}-chunk-{1}c`s.i|]$\}}t|f|fqS((tcov_corr_chunk(R RYR(RJR((s2lib/python2.7/site-packages/dask/dataframe/core.pys s s{0}-combine-{1}-iis{0}-{1}R_RRiN(RRRRRKRRtRURDR/RdRbRRoRRRtcov_corr_combinet cov_corr_aggR5R`R^R=R(RRUR(RlRRR!RhtprefixR!RRRRRYRiRRRj((RJR(s2lib/python2.7/site-packages/dask/dataframe/core.pyR%sF       (6 %1cC`s|jd|jdf}tj|}tj|}|jddt}xXt|D]J\}}||j}||jj||<||j j||>> df = pd.DataFrame({'a': [1, 2, 3, 4, 5, 6], ... 'b': [2, 3, 4, 5, 6, 7]}) >>> a, b = pd_split(df, [0.5, 0.5], random_state=123) # roughly 50/50 split >>> a a b 1 2 3 2 3 4 5 6 7 >>> b a b 0 1 2 3 4 5 4 5 6 (R}RRURRT(RtpRRRY((s2lib/python2.7/site-packages/dask/dataframe/core.pyR vs c`sd|tkr jdStrtjddjdf}jre|gS|fdjDdjSSdS(s take last row (Series) of DataFrame / last value of Series considering NaN. Parameters ---------- a : pd.DataFrame or pd.Series skipna : bool, default True Whether to exclude NaN cS`svxotdtdt|dD]'}|j| }tj|s#|Sq#W||j}|jsr|jdSdS(Nii i( RRRURTRRRtnotnaRR(RwRYtvaltnonnull((s2lib/python2.7/site-packages/dask/dataframe/core.pyt _last_valids)  iiic`s#i|]}||qS(((R R (R9RJ(s2lib/python2.7/site-packages/dask/dataframe/core.pys s RN(RRTREReRRDR(RJR^t series_typ((R9RJs2lib/python2.7/site-packages/dask/dataframe/core.pyRs   &   cC`st|ttfs$tdnt|}|t|krQtdnt|d ttt|d krd}t|ndS(Ns"New division must be list or tuplesNew division must be sortedis8New division must be unique, except for the last element(RKR}RRRrRUR (RkR((s2lib/python2.7/site-packages/dask/dataframe/core.pyRps ,cC`st|t|dkr+tdn|r|d|dkrZd}t|n|d|dkrd}t|qnR|d|dkrd}t|n|d|dkrd}t|nd }|dg}t} |d} d \} } d} ||}xt| t|kr| t|kr|| || krtj|| d f| || tf| || f<|| } | d 7} n|| || krtj|| d f| || tf| || f<|| } | d 7} ntj|| d f| || tf| || f<|| } t|| d ksd|| || d krq| d 7} n| d 7} |j| | d 7} q"W|d|dks|d|d kr>xt| t|D]c}t|d}tj||f| ||tf| || f<||} |j| | d 7} qWnj|r| t|krtj|| d f|| || tf| || f<| d 7} n|j|d| || d fd t f| || d f>> repartition_divisions([1, 3, 7], [1, 4, 6, 7], 'a', 'b', 'c') # doctest: +SKIP {('b', 0): (, ('a', 0), 1, 3, False), ('b', 1): (, ('a', 1), 3, 4, False), ('b', 2): (, ('a', 1), 4, 6, False), ('b', 3): (, ('a', 1), 6, 7, False) ('c', 0): (, (, [('b', 0), ('b', 1)])), ('c', 1): ('b', 2), ('c', 2): ('b', 3)} is+New division must be longer than 2 elementsisHleft side of the new division must be equal or smaller than old divisionisHright side of the new division must be equal or larger than old divisions0left side of old and new divisions are differents1right side of old and new divisions are differentcS`s$t|dko#|d|dkS(s0Whether last division only contains single labeliii(RU(R((s2lib/python2.7/site-packages/dask/dataframe/core.pyt_is_single_last_divsiis=check for duplicate partitions old: %s new: %s combined: %s(ii(ii( RpRURRR6RRRRRWRRV(RJRRitout1tout2R)RR;RRtlowRYRR!t last_elemt_jRttmp((s2lib/python2.7/site-packages/dask/dataframe/core.pytrepartition_divisionss       '0  0  0 .   (,  4 -  S7(cC`s#t|jdtjs(tdny|jdj|}Wntk rb|jd}nXtjd|d|jdd|j}t |s|jd|jdg}n\|d|jdkr|j |jdn|d|jdkr|jdg|}n|j d|S(s7 Repartition a timeseries dataframe by a new frequency is0Can only repartition on frequency for timeseriesRRiR*Rk( RKRkRRt TimestampRctceilRt date_rangeR$RURR.(RR*RRk((s2lib/python2.7/site-packages/dask/dataframe/core.pyR/Ks    cC`s d|t|f}|j|kr)|S|j|kr0|j|}gt|dD]}t||^qV}i}xdt|D]V}tjgt||||dD]}|j|f^qf}||||f>> df = df.repartition([0, 5, 10, 20]) # doctest: +SKIP Also works on Pandas objects >>> ddf = dd.repartition(df, [0, 5, 10, 20]) # doctest: +SKIP srepartition-split-srepartition-merge-R)R_srepartition-dataframe-i(tshard_df_on_indexic3`s'|]\}}|f|fVqdS(N((R RYR(Ri(s2lib/python2.7/site-packages/dask/dataframe/core.pys ss Data must be DataFrame or SeriesN(R/RKRRBRkRbR5R`RRfRERFtutilsRTRRR( RRkR)RRARRhRRTR((Ris2lib/python2.7/site-packages/dask/dataframe/core.pyR.s    "cK`s,|||}t|r(|jjS|S(N(RFRR(RRRR~((s2lib/python2.7/site-packages/dask/dataframe/core.pyRscK`sMt|tr!tj|}n|||}t|rI|jjS|S(N(RKR}RRRSRFRR(RRRR~((s2lib/python2.7/site-packages/dask/dataframe/core.pyR scK`s.t|tr!tj|}n|||S(N(RKR}RRRS(RRR((s2lib/python2.7/site-packages/dask/dataframe/core.pyRscC`s|dkrdnd}t|dkr]t||d|}t||d|}ntjgdd}}t|rtji|d6|d 6Stji|gd6|gd 6S( NRkRRiR^Rwti8RR(RURRRRSRFR(RRmR^tminmaxRR((s2lib/python2.7/site-packages/dask/dataframe/core.pyRns cC`s|dkrdnd}t|dkrx|jd}t|j|d|g}t|j|d|g}ntjgdd}}tji|d6|d 6S( NRkRRiRR^RwRVR(RURoRRRRRSR(RRmR^RWRR((s2lib/python2.7/site-packages/dask/dataframe/core.pyt idxmaxmin_rows!cC`sMt|dkr|S|jddjtd|d|jdddtS(NiRRmR^iR(RURRRXRRW(RRmR^((s2lib/python2.7/site-packages/dask/dataframe/core.pyRps cC`sUt||d|d}t|dkr:tdn|rH|dSd|_|S(NR^Ris*attempt to get argmax of an empty sequence(RpRURRRi(RRmR^RlR((s2lib/python2.7/site-packages/dask/dataframe/core.pyRo s cC`sP|jd|}t||krLd}tj|j|t|n|S(NRsInsufficient elements for `head`. {0} elements requested, only {1} elements available. Try passing larger `npartitions` to `head`.(RRUR+R,Rd(RRtrR((s2lib/python2.7/site-packages/dask/dataframe/core.pyRs "cC`st|tr'tjjj|}nt|tjr_|jsUt|d r_|j S|j rtj t t |jd|j}|j|d|j}t||j|j|j|S|S(sMaybe shift divisions by periods of size freq Used to shift the divisions for the `shift` method. If freq isn't a fixed size (not anchored or relative), then the divisions are shifted appropriately. Otherwise the divisions are cleared. Parameters ---------- df : dd.DataFrame, dd.Series, or dd.Index periods : int The number of periods to shift. freq : DateOffset, timedelta, or time rule string The frequency to shift by. tdeltaRR*(RKRRRRRRt DateOffsetRR|RRRSRRURkR[RReRaRbRf(RRZR*RRk((s2lib/python2.7/site-packages/dask/dataframe/core.pyR]s  '"cK`s4tjtjdg}ttj|d||S(Nt2000Rj(RRRSRCRt to_datetime(R RRj((s2lib/python2.7/site-packages/dask/dataframe/core.pyR]<stnsRc C`sCtjtjdd|g}ttj|d|d|d|S(NitunitRRj(RRRSt TimedeltaRt to_timedelta(R R_RRj((s2lib/python2.7/site-packages/dask/dataframe/core.pyRaBs!RcC`sttj|S(N(RRRR(R ((s2lib/python2.7/site-packages/dask/dataframe/core.pyRJscC`sst|d}t|r:t|r1d}qId}nt|j}tj|gdg|d|d|jS(s1A helper for creating the ``_repr_data`` propertyiscategory[known]scategory[unknown]s...RRi(RURARBRRwRRRSRi(RwRRRw((s2lib/python2.7/site-packages/dask/dataframe/core.pyROs    RcC`stS(N(RS(R&((s2lib/python2.7/site-packages/dask/dataframe/core.pytget_parallel_type_series_scC`stS(N(R(R&((s2lib/python2.7/site-packages/dask/dataframe/core.pytget_parallel_type_dataframedscC`stS(N(R(R&((s2lib/python2.7/site-packages/dask/dataframe/core.pytget_parallel_type_indexiscC`stS(N(R^(R~((s2lib/python2.7/site-packages/dask/dataframe/core.pytget_parallel_type_objectnscC`s t|jS(N(RRf(R~((s2lib/python2.7/site-packages/dask/dataframe/core.pytget_parallel_type_framesscC`stdtjjDS(Ncs`s'|]\}}|tk r|VqdS(N(Re(R R!R~((s2lib/python2.7/site-packages/dask/dataframe/core.pys ys (RRt_lookupR(((s2lib/python2.7/site-packages/dask/dataframe/core.pytparallel_typesxscC`st|t|tS(s4 Does this object have a dask dataframe equivalent? (RRKRh(R((s2lib/python2.7/site-packages/dask/dataframe/core.pyRQ}s c C`sht|r%t|||||St|rKddlj}tjft|dftd|j dD}t|dkr)|j |}t |t r|dd|j d<|jd |_q)d t|d}xDtt|dD])}|j||f|||f|sRRiRHRw(R(i(RQRR't dask.arrayRRMRRURRtlayersRKR-tnew_axestoutput_indicesRRR*Rw( RhRiRjRkRORHRRRY((s2lib/python2.7/site-packages/dask/dataframe/core.pyRs    *"c O`shg}i}x7|D]/}t|trT|j|jdg|jf||j>> subgraph = partitionwise_graph(function, x, y, z=123) # doctest: +SKIP >>> layer = partitionwise_graph(function, df, x, z=123) # doctest: +SKIP >>> graph = HighLevelGraph.from_collections(name, layer, dependencies=[df, x]) # doctest: +SKIP >>> result = new_dd_object(graph, name, metadata, df.divisions) # doctest: +SKIP See Also -------- map_partitions RYiiRitijs/Can't add multi-dimensional array to dataframest numblocksR%(iN(RKRRNRbRR^R*RCRiRRnRR,RW(RRiRXRRxRnR ((s2lib/python2.7/site-packages/dask/dataframe/core.pyRs(#  cC`st|r.d|jjjD}n-t|rU|jt|jf}nd}d}|r~|dt|7}n|S(sS Provide an informative message when the user is asked to provide metadata cS`s%i|]\}}t||qS((R(R R!R~((s2lib/python2.7/site-packages/dask/dataframe/core.pys s s< You did not provide metadata, so Dask is running your function on a small dataset to guess output types. It is possible that Dask will guess incorrectly. To provide an explicit output types or to silence this message, please provide the `meta=` keyword, as described in the map or apply function that you are using.s8 Before: .apply(func) After: .apply(func, meta=%s) N( RERgtto_dictRRFRiRRwR(Rtmeta_strR((s2lib/python2.7/site-packages/dask/dataframe/core.pyR#s " (t __future__RRRRR+t functoolsRRtnumbersRRRtpprintRtnumpyRMtpandasRRt pandas.utilR R ttoolzR R R RRtchestRtCachet ImportErrorRRRRORRURRRRt compatibilityRRRRRRRRRRR R!R"R#R$R%R&R'R(R)t array.coreR*R+R,R-tbaseR.R/R0R1R2R3R4thighlevelgraphR5R6RR7R8t categoricalR9R:R;R<R=R>R?R@RARBRCRDRERFRGRHRt set_optionRR[R]R^RRRcRSRRRbRtand_RRRRRRRRRtnegtor_RRRRtxorRt_bind_operatorRiRRRR!RRR`RRRRR RRRRXRRRRRRR%RRRR RWRRpRBR/R-R.RR RRnRXRpRoRR]R]RaR|RRRtregisterRbRcRdR#ReRfRhRQRRR#(((s2lib/python2.7/site-packages/dask/dataframe/core.pyts    (  .R"X  s #x oo"    [ (         i  -  _D #   '  G/            ;