ó ¦–Õ\c@`sêdZddlmZmZmZddlZddlZddlmZm Z m Z ddl m Z m Z mZmZddlZddlZddlmZmZddlmZdd lmZdd lmZd d lmZmZmZd d lmZd dl m!Z!ddl"m#Z#d„Z$d„Z%d„Z&d„Z'd„Z(ddd„Z*d„Z+d„Z,d„Z-d„Z.d„Z/d„Z0d„Z1ddd„Z2d„Z3de4fd „ƒYZ5d!„Z6d"„Z7d#„Z8dS($sÚ The rechunk module defines: intersect_chunks: a function for converting chunks to new dimensions rechunk: a function to convert the blocks of an existing dask array to new chunks or blockshape i(tabsolute_importtdivisiontprint_functionN(tproducttchaintcount(tgetitemtaddtmult itemgetter(t accumulatetreducei(ttokenize(tHighLevelGraph(t parse_bytesi(t concatenate3tArraytnormalize_chunks(t validate_axis(tempty(tconfigc C`sDg|D]9}tt|fdt|ƒttd|ƒƒƒ^qS(sê Internal utility for cumulative sum with label. >>> cumdims_label(((5, 3, 3), (2, 2, 1)), 'n') # doctest: +NORMALIZE_WHITESPACE [(('n', 0), ('n', 5), ('n', 8), ('n', 11)), (('n', 0), ('n', 2), ('n', 4), ('n', 5))] ii(i(ttupletziptlenR R(tchunkstconsttbds((s1lib/python2.7/site-packages/dask/array/rechunk.pyt cumdims_labelscC`s tt||dtdƒƒƒS(sG >>> new = cumdims_label(((2, 3), (2, 2, 1)), 'n') >>> old = cumdims_label(((2, 2, 1), (5,)), 'o') >>> _breakpoints(new[0], old[0]) (('n', 0), ('o', 0), ('n', 2), ('o', 2), ('o', 4), ('n', 5), ('o', 5)) >>> _breakpoints(new[1], old[1]) (('n', 0), ('o', 0), ('n', 2), ('n', 4), ('n', 5), ('o', 5)) tkeyi(RtsortedR (tcumoldtcumnew((s1lib/python2.7/site-packages/dask/array/rechunk.pyt _breakpoints)s c C`s#d}d}d}g}g}xètdt|ƒƒD]Ñ}||\}}||d\} } | dkr‰|r‰|j|ƒg}q‰n| dkržd}n|}|| |} | }|| krÊq4n|j|t|| ƒfƒ|dkr4|d7}d}q4q4W|r|j|ƒn|S(sB Internal utility to intersect chunks for 1d after preprocessing. >>> new = cumdims_label(((2, 3), (2, 2, 1)), 'n') >>> old = cumdims_label(((2, 2, 1), (5,)), 'o') >>> _intersect_1d(_breakpoints(old[0], new[0])) # doctest: +NORMALIZE_WHITESPACE [[(0, slice(0, 2, None))], [(1, slice(0, 2, None)), (2, slice(0, 1, None))]] >>> _intersect_1d(_breakpoints(old[1], new[1])) # doctest: +NORMALIZE_WHITESPACE [[(0, slice(0, 2, None))], [(0, slice(2, 4, None))], [(0, slice(4, 5, None))]] Parameters ---------- breaks: list of tuples Each tuple is ('o', 8) or ('n', 8) These are pairs of 'o' old or new 'n' indicator with a corresponding cumulative sum. Uses 'o' and 'n' to make new tuples of slices for the new block crosswalk to old blocks. iitnto(trangeRtappendtslice( tbreakststarttlast_endtold_idxtrettret_nexttidxtlabeltbrt last_labeltlast_brtend((s1lib/python2.7/site-packages/dask/array/rechunk.pyt _intersect_1d7s4         cC`sðg|D]"}td„|Dƒƒs|^q}g|D]"}td„|Dƒƒs6|^q6}g|D]}td„|Dƒƒ^qe}g|D]}td„|Dƒƒ^qŽ}t|dƒ}t|dƒ}g|D]} t| ƒ^qÕ} g|D]} t| ƒ^qô} | | ks1td| | fƒ‚n||ksLtdƒ‚ngt||ƒD]#} tt| d | d ƒƒ^q\}xdt|ƒD]V\}}|r’gt|ƒD]}|t d d ƒfg^q±}|j ||ƒq’q’W|S( sÍ Helper to build old_chunks to new_chunks. Handles missing values, as long as the missing dimension is unchanged. Examples -------- >>> old = ((10, 10, 10, 10, 10), ) >>> new = ((25, 5, 20), ) >>> _old_to_new(old, new) # doctest: +NORMALIZE_WHITESPACE [[[(0, slice(0, 10, None)), (1, slice(0, 10, None)), (2, slice(0, 5, None))], [(2, slice(5, 10, None))], [(3, slice(0, 10, None)), (4, slice(0, 10, None))]]] cs`s|]}tj|ƒVqdS(N(tmathtisnan(t.0ty((s1lib/python2.7/site-packages/dask/array/rechunk.pys scs`s|]}tj|ƒVqdS(N(R3R4(R5R6((s1lib/python2.7/site-packages/dask/array/rechunk.pys €scs`s|]}tj|ƒVqdS(N(R3R4(R5R6((s1lib/python2.7/site-packages/dask/array/rechunk.pys ‚scs`s|]}tj|ƒVqdS(N(R3R4(R5R6((s1lib/python2.7/site-packages/dask/array/rechunk.pys ƒsR"R!s&Cannot change dimensions from %r to %rs2Chunks must be unchanging along unknown dimensionsiiN( tanytsumRt ValueErrorRR2R t enumerateR#R%tNonetinsert(t old_chunkst new_chunkstxt old_knownt new_knownt n_missingt n_missing2tcmotcmnR"tsumsR!tsums2tcmt old_to_newR,tmissingtitextra((s1lib/python2.7/site-packages/dask/array/rechunk.pyt _old_to_newps$//))  91cC`s5t||ƒ}t|Œ}td„|Dƒƒ}|S(sÆ Make dask.array slices as intersection of old and new chunks. >>> intersections = intersect_chunks(((4, 4), (2,)), ... ((8,), (1, 1))) >>> list(intersections) # doctest: +NORMALIZE_WHITESPACE [(((0, slice(0, 4, None)), (0, slice(0, 1, None))), ((1, slice(0, 4, None)), (0, slice(0, 1, None)))), (((0, slice(0, 4, None)), (0, slice(1, 2, None))), ((1, slice(0, 4, None)), (0, slice(1, 2, None))))] Parameters ---------- old_chunks : iterable of tuples block sizes along each dimension (convert from old_chunks) new_chunks: iterable of tuples block sizes along each dimension (converts to new_chunks) cs`s!|]}tt|ŒƒVqdS(N(RR(R5tcr((s1lib/python2.7/site-packages/dask/array/rechunk.pys °s(RMRR(R=R>RItcross1tcross((s1lib/python2.7/site-packages/dask/array/rechunk.pytintersect_chunks™s c `sÁt|tƒrh‡fd†|jƒDƒ}x:tˆjƒD]&}||kr;ˆj|||>> import dask.array as da >>> x = da.ones((1000, 1000), chunks=(100, 100)) Specify uniform chunk sizes with a tuple >>> y = x.rechunk((1000, 10)) Or chunk only specific dimensions with a dictionary >>> y = x.rechunk({0: 1000}) Use the value ``-1`` to specify that you want a single chunk along a dimension or the value ``"auto"`` to specify that dask can freely rechunk a dimension to attain blocks of a uniform block size >>> y = x.rechunk({0: -1, 1: 'auto'}, block_size_limit=1e8) c`s+i|]!\}}|t|ˆjƒ“qS((Rtndim(R5tctv(R?(s1lib/python2.7/site-packages/dask/array/rechunk.pys Ús cs`s-|]#\}}|dk r!|n|VqdS(N(R;(R5tlctrc((s1lib/python2.7/site-packages/dask/array/rechunk.pys ßstlimittdtypetprevious_chunkss-Provided chunks are not consistent with shape(t isinstancetdicttitemsR#RRRRtlistRRtshapeRXRR9tmapR8R3R4t plan_rechunktitemsizet_compute_rechunk( R?Rt thresholdtblock_size_limitRKRRt new_shapestnewtoldtstepsRS((R?s1lib/python2.7/site-packages/dask/array/rechunk.pytrechunk´s0%   ,  cC`stttt|ƒƒS(N(R RR_R(R((s1lib/python2.7/site-packages/dask/array/rechunk.pyt_number_of_blocks÷scC`stttt|ƒƒS(N(R RR_tmax(R((s1lib/python2.7/site-packages/dask/array/rechunk.pyt_largest_block_sizeûscC`s&ttd„t||ƒDƒƒ}|S(s; Estimate the graph size during a rechunk computation. cs`s+|]!\}}t|ƒt|ƒVqdS(N(R(R5toctnc((s1lib/python2.7/site-packages/dask/array/rechunk.pys s(R RR(R=R>t crossed_size((s1lib/python2.7/site-packages/dask/array/rechunk.pytestimate_graph_sizeÿs cC`sˆg}xu|D]m}ttj||ƒƒ}x9t|ƒD]+}|||}|j|ƒ||8}q9W|dks t‚q Wt|ƒS(sq Minimally divide the given chunks so as to make the largest chunk width less or equal than *max_width*. i(tinttnptceilR#R$tAssertionErrorR(tdesired_chunkst max_widthRRSt nb_dividesRKR!((s1lib/python2.7/site-packages/dask/array/rechunk.pytdivide_to_width s  cC`s8t|ƒ|kr|St|ƒ}t|ƒdkrž|jƒ}t|ƒ}||}||}|||}||||}||f||f||St|ƒ|}t|ƒ|} gtt|ƒdƒD])} || || d| | df^qÕ} tj| ƒt|ƒ} x| dkr$tj| ƒ\}} } | | dkrª| d7} x| | dkr~| d7} qaWtj | | | | | | | fƒq nC| | | | |krítj | | | | | | | fƒq n| | dkst ‚d| | <|| | <| d8} q Wt t d| ƒƒS(s… Minimally merge the given chunks so as to drop the number of chunks below *max_number*, while minimizing the largest width. iiN(RtsettpopR8R#theapqtheapifyR]theappoptheappushRtRtfilterR;(Rut max_numbertdistincttwR!ttotalt desired_widthtwidthtadjusttnmergesRKtheapRtj((s1lib/python2.7/site-packages/dask/array/rechunk.pytmerge_to_numbersB     C   %%  c`sút|ƒ}g|D]}t|ƒ^q‰g|D]}t|ƒ^q2‰d„tt||ƒƒDƒ‰‡‡fd†t|ƒDƒ‰gt|ƒD]}ˆ|dkr•|^q•}‡‡fd†}t|d|ƒ}ttˆƒ} t|ƒ} t } xÁ|D]¹}| ˆ|ˆ|p!d} | |krH||| |<| } qˆ|} t || | ƒ}t |||ƒ}t|ƒt||ƒkr¶|| |<| t|ƒ| } nt } qW| t | ƒksØt‚| |ksêt‚t| ƒ| fS(sÒ Find an intermediate rechunk that would merge some adjacent blocks together in order to get us nearer the *new_chunks* target, without violating the *block_size_limit* (in number of elements). cS`s5i|]+\}\}}t|ƒt|ƒ|“qS((R(R5tdimRmRn((s1lib/python2.7/site-packages/dask/array/rechunk.pys Xs c`s+i|]!}ˆ|ˆ|p d|“qS(i((R5R‹(tnew_largest_widthtold_largest_width(s1lib/python2.7/site-packages/dask/array/rechunk.pys ]s gð?c`sSˆ|}ˆ|}|dkr)d}n|dkrOtj|ƒtj|ƒSdS(Nig•Ö&è .>ig0¸Dð?(Rrtlog(tktgsetbse(tblock_size_effecttgraph_size_effect(s1lib/python2.7/site-packages/dask/array/rechunk.pyRms     Ri(RRkR:RR#RR RR]tFalseRqRxtTrueRlRtR(R=R>RdRRRSR‹tmerge_candidatesRtsorted_candidatestlargest_block_sizeRtmemory_limit_hittnew_largest_block_sizet largest_widtht chunk_limit((R’R“RŒRs1lib/python2.7/site-packages/dask/array/rechunk.pytfind_merge_rechunkLs:        c C`s t|ƒ}t|ƒ}xçt|ƒD]Ù}t||ƒ}||krJPnt||ƒt||ƒkrpq%ntt||ƒ||ƒ}t|||ƒ}t|ƒ|ks¹t‚t|ƒt||ƒkr%t|ƒt||ƒkr%|||tgraph_size_limitRRRR‹t graph_sizeR€RS((s1lib/python2.7/site-packages/dask/array/rechunk.pytfind_split_rechunk•s    8cC`sÊ|ptjdƒ}|p'tjdƒ}t|tƒrHt|ƒ}nt|ƒ}g}g|D]}td„|Dƒƒ^qa}|dks¨t|ƒ s¨t|ƒr³||gS||}t|ƒ} t|ƒ} t || | gƒ}|t |ƒt |ƒ} |} t } x¬t r¾t | |ƒ}|| kr8Pn| rG| }nt | |||ƒ}t|||ƒ\}}|| krˆ| s”||kr˜Pn|j|ƒ|} |sµPnt} qW||gS(s4 Plan an iterative rechunking from *old_chunks* to *new_chunks*. The plan aims to minimize the rechunk graph size. Parameters ---------- itemsize: int The item size of the array threshold: int The graph growth factor under which we don't bother introducing an intermediate step block_size_limit: int The maximum block size (in bytes) we want to produce during an intermediate step Notes ----- No intermediate steps will be planned if any dimension of ``old_chunks`` is unknown. sarray.rechunk-thresholdsarray.chunk-sizecs`s|]}tj|ƒVqdS(N(R3R4(R5R6((s1lib/python2.7/site-packages/dask/array/rechunk.pys Îsi(RtgetRZtstrRRR7tallRlRkRjR•RpR RR$R”(R=R>RaRcRdRRRhR?thas_nanstlargest_old_blocktlargest_new_blocktgraph_size_thresholdtcurrent_chunkst first_passRŸRR™((s1lib/python2.7/site-packages/dask/array/rechunk.pyR`±sJ )%              c`sˆjdkr+tˆjd|dˆjƒSˆj}tˆj|ƒ}tƒ}tƒ}tˆ|ƒ}d|}d|}t ƒ} t jgˆjD]} t | ƒ^q”ddƒ} x.t j | jƒD]} ˆj f| | | sic3`sD|]:\}\}}|jdko;|jˆj||kVqdS(iN(R'tstopR(R5RKtslctind(R?(s1lib/python2.7/site-packages/dask/array/rechunk.pys )scs`s|]}|dkVqdS(iN((R5td((s1lib/python2.7/site-packages/dask/array/rechunk.pys 3st dependencies(tsizeRR^RXRRRQRR[R RRrRtndindextnameRRR#RytflatR:tnextR£RRtRttolistttoolztmergeR tfrom_collectionsR(R?RRRtcrossedtx2t intermediatesttokent merge_namet split_nametsplit_name_suffixesRSt old_blockstindext new_indextnew_idxRORRKRNtold_block_indicestsubdims1t rec_cat_argtrec_cat_arg_flatt rec_cat_indext ind_slicestold_block_indextslicesR²t old_indextlayertgraph((R?s1lib/python2.7/site-packages/dask/array/rechunk.pyRbsL      1 :, t _PrettyBlockscB`s eZd„Zd„ZeZRS(cC`s ||_dS(N(tblocks(tselfRÐ((s1lib/python2.7/site-packages/dask/array/rechunk.pyt__init__AscC`s§g}g}d}xÑ|jD]Æ}|r‡|d|kr‡|dkrzt|ƒdkrz|jd|d fƒ|d}n|d7}q|dkrÕt|ƒdks«t‚|j|d|dfƒg}d}n|j|ƒqW|rD|dkr|jd|fƒqDt|ƒdks&t‚|j|d|dfƒng}xM|D]E\}}|dkr|jt|ƒƒqQ|jd||fƒqQWdj|ƒS(Niiÿÿÿÿis%d*[%s]s | (RÐRR$R;RtR¢tjoin(RÑtrunstruntrepeatsRStparts((s1lib/python2.7/site-packages/dask/array/rechunk.pyt__str__Ds4      (t__name__t __module__RÒRØt__repr__(((s1lib/python2.7/site-packages/dask/array/rechunk.pyRÏ?s  cC`s5t|tƒr%td„|Dƒƒs+t‚t|ƒS(sÔ Pretty-format *blocks*. >>> format_blocks((10, 10, 10)) 3*[10] >>> format_blocks((2, 3, 4)) [2, 3, 4] >>> format_blocks((10, 10, 5, 6, 2, 2, 2, 7)) 2*[10] | [5, 6] | 3*[2] | [7] cs`s-|]#}t|tƒp$tj|ƒVqdS(N(RZRqR3R4(R5R?((s1lib/python2.7/site-packages/dask/array/rechunk.pys ss(RZRR£RtRÏ(RÐ((s1lib/python2.7/site-packages/dask/array/rechunk.pyt format_blocksgs  cC`s)t|tƒst‚td„|DƒƒS(sH >>> format_chunks((10 * (3,), 3 * (10,))) (10*[3], 3*[10]) cs`s|]}t|ƒVqdS(N(RÜ(R5RS((s1lib/python2.7/site-packages/dask/array/rechunk.pys ~s(RZRRt(R((s1lib/python2.7/site-packages/dask/array/rechunk.pyt format_chunksxscC`sg|D]}t|ƒ^qS(ss >>> format_plan([((10, 10, 10), (15, 15)), ((30,), (10, 10, 10))]) [(3*[10], 2*[15]), ([30], 3*[10])] (RÝ(tplanRS((s1lib/python2.7/site-packages/dask/array/rechunk.pyt format_plans(9t__doc__t __future__RRRR3R{t itertoolsRRRtoperatorRRRR tnumpyRrR¶R R tbaseR thighlevelgraphR tutilsRtcoreRRRRtwrapRtRRR R2RMRQR;RiRjRlRpRxRŠRR R`RbtobjectRÏRÜRÝRß(((s1lib/python2.7/site-packages/dask/array/rechunk.pytsF  "    9 ) C    4 I  O =(