ó ¦–Õ\c@s´ddlZddlZddlmZmZddlmZddlmZm Z ej dddgƒd„ƒZ e d „Z d „Zd „Zejjd d ƒd„ƒZd„Zejjd d ƒd„ƒZd„Zejjd dƒd„ƒZd„Zd„Zejjd d ƒd„ƒZd„Zd„Zd„Zd„Zejjd dƒd„ƒZd„Zd„Z d„Z!d „Z"d!„Z#d"„Z$d#„Z%dS($iÿÿÿÿN(t ndependenciestorder(tget_deps(taddtinctparamstabcdetedcbacCs|jS(N(tparam(trequest((s4lib/python2.7/site-packages/dask/tests/test_order.pyR scCst|d|ƒ|kS(Ntreverse(tsorted(tLR ((s4lib/python2.7/site-packages/dask/tests/test_order.pytissortedscGsdS(N((targs((s4lib/python2.7/site-packages/dask/tests/test_order.pytfscs×|\‰}}}}t‡fd†tdƒDƒƒ}|jitˆdfˆdff|df6tˆdfˆdff|df6ƒt|ƒ}t|ˆdf|ˆdfƒdksÄt‚t|ˆdf|ˆdfƒdksôt‚t‡fd†tdƒDƒƒ}|jitˆdfˆdff|df6tˆdfˆdff|df6ƒt|ƒ}t|ˆdf|ˆdfƒdks£t‚t|ˆdf|ˆdfƒdksÓt‚dS(Nc3s$|]}ˆ|ftffVqdS(N(R(t.0ti(ta(s4lib/python2.7/site-packages/dask/tests/test_order.pys siiiiic3s$|]}ˆ|ftffVqdS(N(R(RR(R(s4lib/python2.7/site-packages/dask/tests/test_order.pys s(tdicttrangetupdateRRtabstAssertionError(Rtbtctdteto((Rs4lib/python2.7/site-packages/dask/tests/test_order.pyt#test_ordering_keeps_groups_togethers"+& 00"+& 0treasonsCan't please 'em allcCs_|\}}}}}itf|df6tf|df6t|dff|df6t|dff|df6t|dff|df6}t|ƒ}||df||dfks¸t‚itf|df6tf|df6t|dff|df6t|dff|df6t|dff|df6}t|ƒ}||df||dfks[t‚dS(sV b0 b1 b2 | \ / a0 a1 a0 should be run before a1 iiiN(RRR(RRRRRRtdskR((s4lib/python2.7/site-packages/dask/tests/test_order.pyttest_avoid_broker_nodes)s #N &#N csÑ|\‰‰‰}}‡‡fd†dddgDƒ}tˆdff|ˆdf<|j‡‡fd†ddddgDƒƒd|ˆUs iiiics(i|]}tˆdfˆ|f“qS(i(R(RR(RR(s4lib/python2.7/site-packages/dask/tests/test_order.pys Ws iiN(RRRR(RRRRR((RRRs4lib/python2.7/site-packages/dask/tests/test_order.pyttest_base_of_reduce_preferredDs",  cCs |\}}}}}it|dff|df6t|dff|df6t|df|dff|df6t|dff|df6t|dff|df6t|dff|df6t|dff|df6}t|ƒ}||df||dfkst‚dS(sî a1 | a2 | a3 d1 / \ / b1 c1 | | b2 c2 | c3 Prefer b1 over c1 because it won't stick around waiting for d1 to complete iiiN(RRR(RRRRRRRR((s4lib/python2.7/site-packages/dask/tests/test_order.pyttest_avoid_upwards_branching`s" cCs|\}}}}}i t|dff|df6t|dff|df6t|df|dff|df6t|dff|df6tf|df6t|dff|df6t|dff|df6tf|df6t|dff|df6t|dff|df6t|dff|df6t|dff|df6t|dff|df6}t|ƒ}||df||dfks‰t‚dS(s a1 | e2 a2 d2 d3 | | \ / e1 a3 d1 \ / \ / b1 c1 | | b2 c2 | c3 Prefer c1 over b1 because c1 will stay in memory less long while b1 computes iiiN(RRR(RRRRRRRR((s4lib/python2.7/site-packages/dask/tests/test_order.pyt$test_avoid_upwards_branching_complexs " sthis case is ambiguouscCs±|\}}}}}it|||f|6t||f|6t|f|6d|6d|6}t|ƒ}||||kst‚||||ks­||||ks­t‚dS(s@ It's not clear who should run first, e or d 1. d is nicer because it exposes parallelism 2. e is nicer (hypothetically) because it will be sooner released (though in this case we need d to run first regardless) a / | \ . b c | / \ | / e d iiN(RRR(RRRRRRRR((s4lib/python2.7/site-packages/dask/tests/test_order.pyt#test_deep_bases_win_over_dependents¤s D cCs”|\}}}}}id|6t|f|6t|f|6d|6t|f|6}t|ƒ}||||ksvt‚||||kst‚dS(sq c | e b | | d a Prefer longer chains first so we should start with c iN(RRR(RRRRRRRR((s4lib/python2.7/site-packages/dask/tests/test_order.pyttest_prefer_deep»s $ cCs?td„tdƒDƒƒ}t|ƒ\}}t||ƒdS(Ncss-|]#}d|dtd|ffVqdS(sx%siN(R(RR((s4lib/python2.7/site-packages/dask/tests/test_order.pys Ïsi'(RRRR(RRt dependenciest dependents((s4lib/python2.7/site-packages/dask/tests/test_order.pyttest_stacklimitÎsc Cs“|\}}}}}d„tdƒDƒ}t|ƒ}t|ƒ|d×s i tyicSs#i|]\}}|d|“qS(i((RRtk((s4lib/python2.7/site-packages/dask/tests/test_order.pys Ýs (RR tlistRRt enumerateR( RRRRRRRtx_keysRtexpected((s4lib/python2.7/site-packages/dask/tests/test_order.pyttest_break_ties_by_strÔs   cCs8titdfd6tdfd6tddfd6ƒdS( NiR)iR*itz(R*i(R*i(RRR(R((s4lib/python2.7/site-packages/dask/tests/test_order.pyt)test_order_doesnt_fail_on_mixed_type_keysâs c sÈtjdƒ}d \}}|jjd||fddƒ}}xntdƒD]`}|dd…ddd…f|dd…dd…dfjdd ƒ}|jdd ƒ}qMW||dd…dfjddƒ}t|j ƒƒ}t |ƒ‰g|j ƒD]} ˆ| ^q} t‡fd †| Dƒƒt | ƒd ksJt ‚g|j ƒD]} | D]} ˆ| ^qaqW} t‡fd †| Dƒƒt | ƒd ks¬t ‚t| ƒ| ksÄt ‚dS(Ns dask.arrayiictsizetchunksiitaxisic3s%|]}|tˆƒdkVqdS(iN(tlen(RR)(R(s4lib/python2.7/site-packages/dask/tests/test_order.pys ôsic3s%|]}|tˆƒdkVqdS(iN(R6(RR)(R(s4lib/python2.7/site-packages/dask/tests/test_order.pys ÷s(iic(iN(ii(tpytestt importorskiptrandomtnormaltNoneRtcumsumtsumRt__dask_graph__Rt __dask_keys__R6RR ( tdatAtBtorigR)t_R*twRR+R tkk((Rs4lib/python2.7/site-packages/dask/tests/test_order.pyt test_gh_3055ès %H& #202cCsM|\}}}}}id|6d|df6d||df6}t|ƒdS(Niii(R(RRRRRRR((s4lib/python2.7/site-packages/dask/tests/test_order.pyttest_type_comparisions_oküs*cCs|\}}}}}itf|6t|f|6t|f|6t|f|6t|f|6}t|ƒ}||||kst‚||||ks™t‚dS(s± a | d b e \ | / c Prefer to finish d and e before starting b. That way c can be released during the long computations. N(RRR(RRRRRRRR((s4lib/python2.7/site-packages/dask/tests/test_order.pyttest_prefer_short_dependentss D s#This is challenging to do preciselyc s]|\}}}}}g||||gD]}|d^q(\}}} } |||||| ||| g } g‰‡fd†} i | |ƒf|6| |ƒf|6| |ƒf|6| | ƒf| 6| |ƒ||f|6| |ƒ||f|6| |ƒ| f|6| |ƒ||f|6| | ƒ| f| 6} tj| ||| gƒˆ| ksYt‚dS(s  aa / | b d bb dd / \ /| | / a c e cc Prefer to run acb first because then we can get that out of the way ics‡‡fd†}|S(NcsˆjˆƒdS(N(tappend(R(tlogR)(s4lib/python2.7/site-packages/dask/tests/test_order.pyRD)s((R)RD(RK(R)s4lib/python2.7/site-packages/dask/tests/test_order.pyR(sN(tdasktgetR(RRRRRRR)taatbbtcctddR/RR((RKs4lib/python2.7/site-packages/dask/tests/test_order.pyttest_run_smaller_sectionss 5!c s¤|\}}}}}gdD]}||^q\}}} gdD]}||^qB\} } } gdD]}||^qh\} }}| ||| | | ||| g }g‰‡fd†}i || ƒf| 6||ƒ| f|6||ƒ|f|6|| ƒf| 6|| ƒ| |f| 6|| ƒ| f| 6||ƒf|6||ƒ|| f|6|| ƒ|f| 6}t|ƒtj||| | gƒˆ|ks t‚dS(s§ c1 | b1 c2 | /| a1 b2 c3 | /| a2 b3 | a3 Prefer to finish a1 stack before proceding to b2 t123cs‡‡fd†}|S(NcsˆjˆƒdS(N(RJ(R(RKR)(s4lib/python2.7/site-packages/dask/tests/test_order.pyRDWs((R)RD(RK(R)s4lib/python2.7/site-packages/dask/tests/test_order.pyRVsN(RRLRMR(RRRRRRRta1ta2ta3tb1tb2tb3tc1tc2tc3R/RR((RKs4lib/python2.7/site-packages/dask/tests/test_order.pyttest_local_parents_of_reduction<s*&&&   c sÐ|\‰‰}}}gdD]}ˆ|^q\ }}}}}} } } } gdD]}ˆ|^qT\} }}}i tf| 6tf|6tf|6tf|6t| f|6t| f|6t| |f|6t|f|6t||f|6t|f| 6t||f| 6t|f| 6t|f| 6}t|ƒ‰dt‡‡fd†dDƒƒko^dknsit‚dt‡‡fd†dDƒƒko™dkns¤t‚ˆt| |||gƒd ksÌt‚d S( sâ a1 a2 a3 a4 a5 a6 a7 a8 a9 \ | / \ | / \ | / \ | / b1 b2 b3 b4 Want to finish off a local group before moving on. This is difficult because all groups are connected. t 123456789t1234ic3s-|]#}ˆˆ|tˆƒdkVqdS(iN(R6(RR(RR(s4lib/python2.7/site-packages/dask/tests/test_order.pys ‰siic3s-|]#}ˆˆ|tˆƒdkVqdS(iN(R6(RR(RR(s4lib/python2.7/site-packages/dask/tests/test_order.pys ŠsiiN(RRR=Rtmin(RRRDRRTRURVta4ta5ta6ta7ta8ta9RWRXRYtb4R((RRRs4lib/python2.7/site-packages/dask/tests/test_order.pyttest_nearest_neighborks( 8)          ;;cCs[itfd6tfd6tfd 6}t|ƒ}|idd 6dd 6dd 6ksWt‚dS( s% Prefer ordering tasks by name first RiiiiN(Ri(Ri(Ri(Ri(Ri(Ri(RRR(RR((s4lib/python2.7/site-packages/dask/tests/test_order.pyttest_string_orderingŽs $  cCsuitdfd6tdfd 6tdfd 6tfd6}t|ƒ}|idd6dd 6dd 6dd 6ksqt‚dS(s? Prefer ordering tasks by name first even when in dependencies RRiiiiN(Ri(Ri(Ri(Ri(Ri(Ri(RRR(RR((s4lib/python2.7/site-packages/dask/tests/test_order.pyttest_string_ordering_dependents—s*   cCs4|\}}}}}id|df6d|df6d|df6t|df|df|dff|df6d|df6d|df6t|df|df|dff|df6}t|ƒ}||df||dfksät‚||df||dfks t‚||df||dfks0t‚dS(Niii(RRR(RRRRRDRR((s4lib/python2.7/site-packages/dask/tests/test_order.pyttest_prefer_short_narrow¢s   +  . &&cCs›|\}}}}}||}id|6t|ddf|df6t|ddf|df6d|df6t|df|df|dff|df6t|ddf|df6t|ddf|df6t|df|df|dff|df6}t|ƒ}||df||dfks%t‚||df||dfksKt‚||df||dfksqt‚||df||dfks—t‚dS(s˜ From https://github.com/dask/dask-ml/issues/206#issuecomment-395869929 Two cases, one where chunks of an array are independent, and one where the chunks of an array have a shared source. We handled the independent one "well" earlier. Good: c2 / \ \ / \ \ c1 \ \ / | \ \ \ c0 a0 b0 a1 b1 Bad: c2 / \ \ / \ \ c1 \ \ / | \ \ \ c0 a0 b0 a1 b1 \ \ / / \ \ / / a-b The difference is that all the `a` and `b` tasks now have a common ancestor. We would like to choose c1 *before* a1, and b1 because * we can release a0 and b0 once c1 is done * we don't need a1 and b1 to compute c1. iiiN(RRR(RRRRRDtabRR((s4lib/python2.7/site-packages/dask/tests/test_order.pyttest_prefer_short_ancestor´s &  +. &&&cCs|\}}}}}itf|df6t|dff|df6t|dff|df6t|df|dff|df6tf|df6t|df|df|dff|df6tf|df6t|dff|df6t|dff|df6t|df|df|dff|df6tf|df6t|df|df|dff|df6tf|df6t|dff|df6t|dff|df6t|df|dff|df6}t|ƒ}||df||dfks||df||dfkst‚dS(sÞ b1 b3 b5 |\ / | \ / | c1 c2 c3 c4 c5 |/ | \ | / | \| d1 d2 d3 d4 d5 | | | e1 e2 e5 Want to finish b1 before we start on e5 iiiiiN(RRR(RRRRRRRR((s4lib/python2.7/site-packages/dask/tests/test_order.pyttest_map_overlapïs( "+++% (&R7RLt dask.orderRRt dask.coreRtdask.utils_testRRtfixtureRtFalseR RRtmarktxfailR R!R"R#R$R%R(R0R2RGRHRIRRR]RhRiRjRkRmRn(((s4lib/python2.7/site-packages/dask/tests/test_order.pyts8  !     %      & / #  ;