ó ßüÚ\c@`sFddlmZmZmZddlmZddlmZddlZddl m Z ddl m Z ddl mZddlmZmZmZmZd d lmZd d lmZmZe jeddd „ƒZd efd„ƒYZdgZe jeedd„ƒZ e!e"e#e$fZ%e&dd„Z'ed„Z(dS(i(tprint_functiontdivisiontabsolute_import(t defaultdict(tcycleN(tgen(tReturn(tSubgraphCallable(tmergetconcattgroupbytdropi(trpc(tAllttokeyc#`sXddlm‰tƒ}tƒ}|‰d„|jƒDƒ}tƒ}tƒ}xĐt|ƒt|ƒt|ƒkr"ttƒ} tƒ} tƒ} x…|jƒD]w\} } | |kržq ny8tj t| |ƒƒ}| |j | ƒ|| | 's c`si|]}ˆ|ƒ|“qS(((Rtaddr(R (s5lib/python2.7/site-packages/distributed/utils_comm.pys ;s c `s=i|]3\}}ˆˆ||dˆdˆdtƒ|“qS(twhot serializerstmax_connections(tFalse(Rtaddresstkeys(RR RR(s5lib/python2.7/site-packages/distributed/utils_comm.pys >s tdataNc`s(h|]\}}|ˆkr|’qS(((RRR(tresponse(s5lib/python2.7/site-packages/distributed/utils_comm.pys Ts c`s#i|]}tˆ|ƒ|“qS((tlist(RR(toriginal_who_has(s5lib/python2.7/site-packages/distributed/utils_comm.pys Ws (tworkerRRtitemstdicttlenRRtrandomtchoicetappendt IndexErrortaddtEnvironmentErrortupdatetvaluest close_rpcR(twho_hasR tcloseRRt bad_addressestmissing_workerstresultst all_bad_keystdtrevtbad_keystkeyt addressesRtrpcst coroutinesRtctr((RRRR RRs5lib/python2.7/site-packages/distributed/utils_comm.pytgather_from_workerssN    %         t WrappedKeycB`s eZdZd„Zd„ZRS(s• Interface for a key in a dask graph. Subclasses must have .key attribute that refers to a key in a dask graph. Sometimes we want to associate metadata to keys in a dask graph. For example we might know that that key lives on a particular machine or can only be accessed in a certain way. Schedulers may have particular needs that can only be addressed by additional metadata. cC`s ||_dS(N(R5(tselfR5((s5lib/python2.7/site-packages/distributed/utils_comm.pyt__init__fscC`sdt|ƒj|jfS(Ns%s('%s')(ttypet__name__R5(R=((s5lib/python2.7/site-packages/distributed/utils_comm.pyt__repr__is(R@t __module__t__doc__R>RA(((s5lib/python2.7/site-packages/distributed/utils_comm.pyR<[s  c #`sľt|tƒst‚t|tƒs*t‚ttd„|jƒDƒƒƒ}tt|jƒŒƒ\}}ttdt |ƒt |ƒƒ}tdct |ƒ7}sicS`s)i|]\}}d„|Dƒ|“qS(cS`s"i|]\}}}||“qS(((Rt_R5tvalue((s5lib/python2.7/site-packages/distributed/utils_comm.pys …s ((RRR((s5lib/python2.7/site-packages/distributed/utils_comm.pys …s c`si|]}ˆ|ƒ|“qS(((RR(R (s5lib/python2.7/site-packages/distributed/utils_comm.pys ‡s RtreportRNcs`s|]}|dVqdS(tnbytesN((Rto((s5lib/python2.7/site-packages/distributed/utils_comm.pys •scS`s;i|]1\}}g|D]\}}}|^q|“qS(((RRRRDRF((s5lib/python2.7/site-packages/distributed/utils_comm.pys —s i(t isinstanceR!tAssertionErrorRR R tzipR t_round_robin_counterR"RR R t update_dataR*R+RR(tncoresRR RHRtworkerstnamest worker_itertLR2R7RRtoutR:RIR,((R s5lib/python2.7/site-packages/distributed/utils_comm.pytscatter_to_workersps& "#Fc`sFˆdkr1tƒ‰t|ˆˆƒ}|ˆfSt|ƒ}|tkr^|sS|St|dƒtkr>|d}tƒ‰‡‡fd†|jjƒDƒ}t‡‡fd†|dDƒƒ}ˆr7ˆjˆƒˆrđtd„ˆDƒƒntd„ˆDƒƒ‰|j ˆ}t||j ||j ƒf|ˆS|Sq^t‡‡fd†|DƒƒSn|t krŁ|st|Sg|D]} t| ˆˆƒ^q{} || ƒS|t kr|růg|jƒD]} t| ˆˆƒ^qÂ} t t|jƒ| ƒƒS|SnBt|tƒr>|j} ˆr-t| ƒ} nˆj|ƒ| S|SdS( ső Unpack WrappedKey objects from collection Returns original collection and set of all found WrappedKey objects Examples -------- >>> rd = WrappedKey('mykey') >>> unpack_remotedata(1) (1, set()) >>> unpack_remotedata(()) ((), set()) >>> unpack_remotedata(rd) ('mykey', {WrappedKey('mykey')}) >>> unpack_remotedata([1, rd]) ([1, 'mykey'], {WrappedKey('mykey')}) >>> unpack_remotedata({1: rd}) ({1: 'mykey'}, {WrappedKey('mykey')}) >>> unpack_remotedata({1: [rd]}) ({1: ['mykey']}, {WrappedKey('mykey')}) Use the ``byte_keys=True`` keyword to force string keys >>> rd = WrappedKey(('x', 1)) >>> unpack_remotedata(rd, byte_keys=True) ("('x', 1)", {WrappedKey('('x', 1)')}) ic`s+i|]!\}}t|ˆˆƒ|“qS((tunpack_remotedata(RRR(t byte_keystfutures(s5lib/python2.7/site-packages/distributed/utils_comm.pys Čs c3`s!|]}t|ˆˆƒVqdS(N(RW(Rti(RXRY(s5lib/python2.7/site-packages/distributed/utils_comm.pys Ęsics`s|]}t|jƒVqdS(N(RR5(Rtf((s5lib/python2.7/site-packages/distributed/utils_comm.pys Îscs`s|]}|jVqdS(N(R5(RR[((s5lib/python2.7/site-packages/distributed/utils_comm.pys Đsc3`s!|]}t|ˆˆƒVqdS(N(RW(Rtitem(RXtmyset(s5lib/python2.7/site-packages/distributed/utils_comm.pys ŰsN(tNoneRRWR?ttupleRtdskR R)tinkeystoutkeytnametcollection_typesR!R*RMRt issubclassR<R5RR'(RJRXR]RUttyptscR`targsRaR\toutsRR*R((RXRYR]s5lib/python2.7/site-packages/distributed/utils_comm.pyRWŸsL       "#  $  %  +  c`s°t|ƒ}y't|ˆƒr2|ˆkr2ˆ|SWntk rFnX|tkr|g|D]}t|ˆdˆƒ^q]ƒS|tkr¨‡‡fd†|jƒDƒS|SdS(sË Merge known data into tuple or dict Parameters ---------- o: core data structures containing literals and keys d: dict mapping of keys to data Examples -------- >>> data = {'x': 1} >>> pack_data(('x', 'y'), data) (1, 'y') >>> pack_data({'a': 'x', 'b': 'y'}, data) # doctest: +SKIP {'a': 1, 'b': 'y'} >>> pack_data({'a': ['x'], 'b': 'y'}, data) # doctest: +SKIP {'a': [1], 'b': 'y'} t key_typesc`s.i|]$\}}t|ˆdˆƒ|“qS(Rj(t pack_data(RRR(R2Rj(s5lib/python2.7/site-packages/distributed/utils_comm.pys s N(R?RKt TypeErrorRdRkR!R (RJR2RjRftx((R2Rjs5lib/python2.7/site-packages/distributed/utils_comm.pyRkńs    , ()t __future__RRRt collectionsRt itertoolsRR#ttornadoRt tornado.genRtdask.optimizationRttoolzRR R R tcoreR tutilsR Rt coroutinetTrueR^R;tobjectR<RNRVR_RRt frozensetRdRRWRk(((s5lib/python2.7/site-packages/distributed/utils_comm.pyts$ "I +R