B T\D@sNddlmZmZmZddlmZddlmZddlm Z ddl m Z m Z m Z mZddlmZddlmZmZdd lmZmZmZmZmZd d lmZd d lmZd d lmZd dlm Z ddZ!d3ddZ"ddZ#d4ddZ$d5ddZ%ddZ&ddZ'ddZ(d d!Z)d"d#Z*d$d%Z+d6d&d'Z,d(d)Z-d*d+Z.d7d-d.Z/d/d0Z0d1d2Z1dS)8)absolute_importdivisionprint_function)getitem)product)Integral)mergepipeconcatpartial)map)chunkwrap)Array map_blocks concatenate concatenate3 reshapelist)HighLevelGraph)tokenize)flatten)concretecCs|dftdd|ddD}g}xtt|dd|ddD]\}\}}||d}||kr||tdddqH||kr|td|qH||kr|dkr|tddqH|t| dqHWt|}tdd|Dr|St||fSdS)a >>> fractional_slice(('x', 5.1), {0: 2}) # doctest: +SKIP (getitem, ('x', 6), (slice(0, 2),)) >>> fractional_slice(('x', 3, 5.1), {0: 2, 1: 3}) # doctest: +SKIP (getitem, ('x', 3, 5), (slice(None, None, None), slice(-3, None))) >>> fractional_slice(('x', 2.9, 5.1), {0: 2, 1: 3}) # doctest: +SKIP (getitem, ('x', 3, 5), (slice(0, 2), slice(-3, None))) rcss|]}tt|VqdS)N)intround).0ir1lib/python3.7/site-packages/dask/array/overlap.py sz#fractional_slice..r Ncss|]}|tdddkVqdS)N)slice)rindrrrr .s)tuple enumeratezipgetappendr!allr)ZtaskaxesZroundedindexrtrdepthrrrfractional_slices $, r.Nc sfddg}xRt|ddD]>\}}d}|dkr>|d7}||dkrV|d7}||q"Wfddt|ddD}|dk r|gg|}tt|} fddt|D} t| | } | S) a Get all neighboring keys around center Parameters ---------- k: tuple They key around which to generate new keys dims: Sequence[int] The number of chunks in each dimension name: Option[str] The name to include in the output keys, or none to include no name axes: Dict[int, int] The axes active in the expansion. We don't expand on non-active axes Examples -------- >>> expand_key(('x', 2, 3), dims=[5, 5], name='y', axes={0: 1, 1: 1}) # doctest: +NORMALIZE_WHITESPACE [[('y', 1.1, 2.1), ('y', 1.1, 3), ('y', 1.1, 3.9)], [('y', 2, 2.1), ('y', 2, 3), ('y', 2, 3.9)], [('y', 2.9, 2.1), ('y', 2.9, 3), ('y', 2.9, 3.9)]] >>> expand_key(('x', 0, 4), dims=[5, 5], name='y', axes={0: 1, 1: 1}) # doctest: +NORMALIZE_WHITESPACE [[('y', 0, 3.1), ('y', 0, 4)], [('y', 0.9, 3.1), ('y', 0.9, 4)]] csNg}|ddkr||d|||d|dkrJ||d|S)Ng?rr )r')rr"rv)dimsrrindsMs  zexpand_key..indsr Nrcs,g|]$\}}|dr"||n|gqS)r)r&)rrr")r)r1rr _szexpand_key..cs$g|]\}}|dr|ndqS)rr )r&)rrd)r)rrr2cs)r$r'listrr) kr0namer)shaperr"ZnumargsseqZshape2resultr)r)r0r1r expand_key4s      r;c Csttt|j}tt||d}t|tt|ttt t}dt ||}dt ||}i}i}xv|D]n} t |j f| |} |j f| | kr| ||f| <qf|j f| ||f| <t t|d| |dff||f| <qfWg} xt|jD]\} } t| dkr | | q| d|| dg}| d|| dg}g}x0| ddD] }|||| dd qHW| |||qWt||}tj|||gd }t||| |jd S) aK Share boundaries between neighboring blocks Parameters ---------- x: da.Array A dask array axes: dict The size of the shared boundary per axis The axes input informs how many cells to overlap between neighboring blocks {0: 2, 2: 5} means share two cells in 0 axis, 5 cells in 2 axis )r0r)zoverlap-zgetitem-)N)r6r rr)Z dependencies)dtype)r4r lenchunksr r;r Z __dask_keys__rr rr.r6rrr$r'r&rrZfrom_collectionsrr=)xr)r0Z expand_key2Z interior_keysr6Z getitem_nameZinterior_slicesZoverlap_blocksr5Z frac_slicer?rZbdsleftrightZmidbdZdskZgraphrrroverlap_internalhs8 $   rDcCs&t|j|}t|j|}t|||dS)zTrim sides from each block. This couples well with the ``map_overlap`` operation which may leave excess data on each block. See also -------- dask.array.overlap.map_overlap )r)boundary) coerce_depthndimcoerce_boundary trim_internal)r@r-rEr) boundary2rrr trim_overlaps  rKc Cst|j|}g}xt|jD]\}}||d}g}x|t|D]p\}} |dkrd| ||dd} n@|dkr|| ||dn| } |t|dkr| ||dn| } || q>W|t|qWt|} tt t ||d|| |j dS)z Trim sides from each block This couples well with the overlap operation, which may leave excess data on each block See also -------- dask.array.chunk.trim dask.array.map_blocks nonerrr )r)rE)r?r=) rHrGr$r?r&r>r'r#rr _trimr=) r@r)rEZolistrrCZbdyZilistjr3r?rrrrIs  $rIcsfddt|jDddD}fddtt|ddD}fddtt|dd |dd|D}|td dt||DS) zSimilar to dask.array.chunk.trim but requires one to specificy the boundary condition. ``axes``, and ``boundary`` are assumed to have been coerced. csg|]}|dqS)r)r&)rr)r)rrr2sz_trim..css|]}|r| ndVqdS)Nr)raxrrrr sz_trim..c3s6|].\}\}}|dkr*|ddkr*dn|VqdS)rrLN)r&)rrchunk_locationrO)rErrr srzchunk-locationc3s<|]4\}\}}}||dkr0|ddkr0dn|VqdS)r rLN)r&)rrr?rPrO)rErrr sz num-chunkscss|]\}}t||VqdS)N)r!)rfrontZbackrrrr s)rangerGr$r%r#)r@r)rEZ block_infoZ axes_backZ trim_frontZ trim_backr)r)rErrMs      rMcCstdddf|td|ftdddf|j|d}tdddf|t| dftdddf|j|d}||}||}t||||\}}t|||g|dS)zv Copy a slice of an array around to its other side Useful to create periodic boundary conditions for overlap Nrr )axis)r!rG_remove_overlap_boundariesr)r@rSr-rArBlr,rrrperiodicsrVcCs|dkrDtdddf|tddftdddf|j|d}n@tdddf|t|dddftdddf|j|d}tdddf|td| ddftdddf|j|d}||}||}t||||\}}t|||g|dS)z\ Reflect boundaries of array on the same side This is the converse of ``periodic`` r Nrr<)rS)r!rGrTr)r@rSr-rArBrUr,rrrreflects "$rWcCstdddf|tddftdddf|j|d}tdddf|tdddftdddf|j|d}t||g||d}t||g||d}t||||\}}t|||g|dS)z Each reflect each boundary value outwards This mimics what the skimage.filters.gaussian_filter(... mode="nearest") does. Nrr r<)rS)r!rGrrT)r@rSr-rArBrUr,rrrnearestsrYcCsHt|j}|f||<tjttt||t||jd}t|||g|dS)z, Add constant slice to either side of array )r?r=)rS) r4r?rZfullr#r sumr=r)r@rSr-valuer?crrrconstant*s   r]cCsLt|j}|f||<t|j}|f||<|t|}|t|}||fS)N)r4r?rechunkr#)rUr,rSr-ZlchunksZrchunksrrrrT5s    rTcstts&tfddt|jDttsLtfddt|jDxt|jD]}|d}|dkrrqX|d}|dkrqXqX|dkrt|||}qX|dkrt|||}qX|dkrt|||}qX|krXt||||}qXW|S) zp Add boundary conditions to an array before overlaping See Also -------- periodic constant c3s|]}|fVqdS)Nr)rr)kindrrr Iszboundaries..c3s|]}|fVqdS)Nr)rr)r-rrr KsrrLrVrWrY) isinstancedictrRrGr&rVrWrYr])r@r-r_rr3Z this_kindr)r-r_r boundaries@s(    rbc st|j|t|j|fddt|jD}x8t||jD](\}}|t|kr>td|t|fq>Wt|}t |}t fdd D}t ||} | S)ao Share boundaries between neighboring blocks Parameters ---------- x: da.Array A dask array depth: dict The size of the shared boundary per axis boundary: dict The boundary condition on each axis. Options are 'reflect', 'periodic', 'nearest', 'none', or an array value. Such a value will fill the boundary with that value. The depth input informs how many cells to overlap between neighboring blocks ``{0: 2, 2: 5}`` means share two cells in 0 axis, 5 cells in 2 axis. Axes missing from this input will not be overlapped. Examples -------- >>> import numpy as np >>> import dask.array as da >>> x = np.arange(64).reshape((8, 8)) >>> d = da.from_array(x, chunks=(4, 4)) >>> d.chunks ((4, 4), (4, 4)) >>> g = da.overlap.overlap(d, depth={0: 2, 1: 1}, ... boundary={0: 100, 1: 'reflect'}) >>> g.chunks ((8, 8), (6, 6)) >>> np.array(g) array([[100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100], [100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100], [ 0, 0, 1, 2, 3, 4, 3, 4, 5, 6, 7, 7], [ 8, 8, 9, 10, 11, 12, 11, 12, 13, 14, 15, 15], [ 16, 16, 17, 18, 19, 20, 19, 20, 21, 22, 23, 23], [ 24, 24, 25, 26, 27, 28, 27, 28, 29, 30, 31, 31], [ 32, 32, 33, 34, 35, 36, 35, 36, 37, 38, 39, 39], [ 40, 40, 41, 42, 43, 44, 43, 44, 45, 46, 47, 47], [ 16, 16, 17, 18, 19, 20, 19, 20, 21, 22, 23, 23], [ 24, 24, 25, 26, 27, 28, 27, 28, 29, 30, 31, 31], [ 32, 32, 33, 34, 35, 36, 35, 36, 37, 38, 39, 39], [ 40, 40, 41, 42, 43, 44, 43, 44, 45, 46, 47, 47], [ 48, 48, 49, 50, 51, 52, 51, 52, 53, 54, 55, 55], [ 56, 56, 57, 58, 59, 60, 59, 60, 61, 62, 63, 63], [100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100], [100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100]]) csg|]}|dqS)r)r&)rr)depth2rrr2szoverlap..zThe overlapping depth %d is larger than your smallest chunk size %d. Rechunk your array with a larger chunk size or a chunk size that more evenly divides the shape of your array.c3s2|]*\}}||ddkr$|dndfVqdS)rLrrN)r&)rr5v)rJrrr szoverlap..)rFrGrHrRr%r?min ValueErrorrbrDraitemsrtrim) r@r-rEZ depth_valuesr3r\Zx2Zx3rhZx4r)rJrcroverlapas4       ric Csx|D]\}}||d}|dkr |dkr t|j}|||<t|j}|f||<tj|||jd}t|j} t| |} | d|7<| d|7<t| | |<t |||g|d}| | }q W|S)a? Pads an array which has 'none' as the boundary type. Used to simplify trimming arrays which use 'none'. >>> import dask.array as da >>> x = da.arange(6, chunks=3) >>> add_dummy_padding(x, {0: 1}, {0: 'none'}).compute() # doctest: +NORMALIZE_WHITESPACE array([..., 0, 1, 2, 3, 4, 5, ...]) rrL)r?r=r<)rS) rgr&r4r7r?remptyr=r#rr^) r@r-rEr5rdr3Z empty_shapeZ empty_chunksrjZ out_chunksZ ax_chunksrrradd_dummy_paddings        rkTc Kst|j|}t|j|}tdd|jDs0tt|||d}tdd|jDsVt|j|f|} tdd| jDs|t|rt| ||S| SdS)aU Map a function over blocks of the array with some overlap We share neighboring zones between blocks of the array, then map a function, then trim away the neighboring strips. Parameters ---------- func: function The function to apply to each extended block depth: int, tuple, or dict The number of elements that each block should share with its neighbors If a tuple or dict then this can be different per axis boundary: str, tuple, dict How to handle the boundaries. Values include 'reflect', 'periodic', 'nearest', 'none', or any constant value like 0 or np.nan trim: bool Whether or not to trim ``depth`` elements from each block after calling the map function. Set this to False if your mapping function already does this for you **kwargs: Other keyword arguments valid in ``map_blocks`` Examples -------- >>> import numpy as np >>> import dask.array as da >>> x = np.array([1, 1, 2, 3, 3, 3, 2, 1, 1]) >>> x = da.from_array(x, chunks=5) >>> def derivative(x): ... return x - np.roll(x, 1) >>> y = x.map_overlap(derivative, depth=1, boundary=0) >>> y.compute() array([ 1, 0, 1, 1, 0, 0, -1, -1, 0]) >>> x = np.arange(16).reshape((4, 4)) >>> d = da.from_array(x, chunks=(2, 2)) >>> d.map_overlap(lambda x: x + x.size, depth=1).compute() array([[16, 17, 18, 19], [20, 21, 22, 23], [24, 25, 26, 27], [28, 29, 30, 31]]) >>> func = lambda x: x + x.size >>> depth = {0: 1, 1: 1} >>> boundary = {0: 'reflect', 1: 'none'} >>> d.map_overlap(func, depth, boundary).compute() # doctest: +NORMALIZE_WHITESPACE array([[12, 13, 14, 15], [16, 17, 18, 19], [20, 21, 22, 23], [24, 25, 26, 27]]) css$|]}|D]}t|tkVq qdS)N)typer)rccr\rrrr szmap_overlap..)r-rEcss$|]}|D]}t|tkVq qdS)N)rlr)rrmr\rrrr scss$|]}|D]}t|tkVq qdS)N)rlr)rrmr\rrrr sN) rFrGrHr(r?AssertionErrorrirrI) r@funcr-rErhkwargsrcrJgZg2rrr map_overlaps7   rrcCs4t|tr|f|}t|tr0ttt||}|S)N)r`rr#rar%rR)rGr-rrrrFs    rFcCsD|dkr d}t|ttfs$|f|}t|tr@ttt||}|S)NrW)r`r#rar%rR)rGrErrrrHs  rH)NN)N)N)NN)NT)2Z __future__rrroperatorr itertoolsrZnumbersrZtoolzrr r r Z toolz.curriedr rrZcorerrrrrZhighlevelgraphrbaserrZutilsrr.r;rDrKrIrMrVrWrYr]rTrbrirkrrrFrHrrrrs8        " 44    !H E