ó ¦–Õ\c@`s!ddlmZmZmZddlmZddlmZddlm Z ddl m Z ddl m Z mZmZddlmZdd lmZed d!ƒZdefd„ƒYZedd"ƒZdefd„ƒYZde fd„ƒYZedd#ƒZdefd„ƒYZd S($i(tabsolute_importtdivisiontprint_function(t namedtuple(tstarmap(t default_timer(tsleep(tProcesstPipetcurrent_processi(tCallback(timport_requiredtTaskDatatkeyttaskt start_timetend_timet worker_idtProfilercB`s_eZdZd„Zd„Zd„Zd„Zd„Zd„Zd„Z d„Z d „Z RS( s4A profiler for dask execution at the task level. Records the following information for each task: 1. Key 2. Task 3. Start time in seconds since the epoch 4. Finish time in seconds since the epoch 5. Worker id Examples -------- >>> from operator import add, mul >>> from dask.threaded import get >>> dsk = {'x': 1, 'y': (add, 'x', 10), 'z': (mul, 'y', 2)} >>> with Profiler() as prof: ... get(dsk, 'z') 22 >>> prof.results # doctest: +SKIP [('y', (add, 'x', 10), 1435352238.48039, 1435352238.480655, 140285575100160), ('z', (mul, 'y', 2), 1435352238.480657, 1435352238.480803, 140285566707456)] These results can be visualized in a bokeh plot using the ``visualize`` method. Note that this requires bokeh to be installed. >>> prof.visualize() # doctest: +SKIP You can activate the profiler globally >>> prof.register() # doctest: +SKIP If you use the profiler globally you will need to clear out old results manually. >>> prof.clear() cC`si|_g|_i|_dS(N(t_resultstresultst_dsk(tself((s7lib/python2.7/site-packages/dask/diagnostics/profile.pyt__init__9s  cC`s|jƒtt|ƒjƒS(N(tcleartsuperRt __enter__(R((s7lib/python2.7/site-packages/dask/diagnostics/profile.pyR>s cC`s|jj|ƒdS(N(Rtupdate(Rtdsk((s7lib/python2.7/site-packages/dask/diagnostics/profile.pyt_startBscC`s'tƒ}||||f|j|Ns( tdictRtitemsRtlistRR tvaluesR(RRRtfailedR((s7lib/python2.7/site-packages/dask/diagnostics/profile.pyt_finishMs$cK`s&ddlm}||j|j|S(Ni(t plot_tasks(tprofile_visualizeR/RR(RtkwargsR/((s7lib/python2.7/site-packages/dask/diagnostics/profile.pyt_plotRscK`sddlm}|||S(sVisualize the profiling run in a bokeh plot. See also -------- dask.diagnostics.profile_visualize.visualize i(t visualize(R0R3(RR1R3((s7lib/python2.7/site-packages/dask/diagnostics/profile.pyR3VscC`s!|jjƒ|j2i|_dS(s#Clear out old results from profilerN(RRRR(R((s7lib/python2.7/site-packages/dask/diagnostics/profile.pyR`s ( t__name__t __module__t__doc__RRRR R$R.R2R3R(((s7lib/python2.7/site-packages/dask/diagnostics/profile.pyRs&        t ResourceDatattimetmemtcputResourceProfilercB`sƒeZdZdd„Zd„Zd„Zd„Zd„Zd„Zd„Z d „Z d „Z e Z d „Z d „Zd „ZRS(sA profiler for resource use. Records the following each timestep 1. Time in seconds since the epoch 2. Memory usage in MB 3. % CPU usage Examples -------- >>> from operator import add, mul >>> from dask.threaded import get >>> dsk = {'x': 1, 'y': (add, 'x', 10), 'z': (mul, 'y', 2)} >>> with ResourceProfiler() as prof: # doctest: +SKIP ... get(dsk, 'z') 22 These results can be visualized in a bokeh plot using the ``visualize`` method. Note that this requires bokeh to be installed. >>> prof.visualize() # doctest: +SKIP You can activate the profiler globally >>> prof.register() # doctest: +SKIP If you use the profiler globally you will need to clear out old results manually. >>> prof.clear() # doctest: +SKIP Note that when used as a context manager data will be collected throughout the duration of the enclosed block. In contrast, when registered globally data will only be collected while a dask scheduler is active. icC`s(||_t|_d|_g|_dS(N(t_dttFalset_enteredtNonet_trackerR(Rtdt((s7lib/python2.7/site-packages/dask/diagnostics/profile.pyRŽs   cC`s|jdk o|jjƒS(N(R@R?tis_alive(R((s7lib/python2.7/site-packages/dask/diagnostics/profile.pyt _is_running”scC`sE|jƒs.t|jƒ|_|jjƒn|jjjdƒdS(Ntcollect(RCt_TrackerR<R@Rt parent_conntsend(R((s7lib/python2.7/site-packages/dask/diagnostics/profile.pyt_start_collect—s cC`sK|jƒrG|jjjdƒ|jjtt|jjjƒƒƒndS(Nt send_data( RCR@RFRGRtextendRR7trecv(R((s7lib/python2.7/site-packages/dask/diagnostics/profile.pyt _stop_collects cC`s0t|_|jƒ|jƒtt|ƒjƒS(N(tTrueR>RRHRR;R(R((s7lib/python2.7/site-packages/dask/diagnostics/profile.pyR¢s   cG`s7t|_|jƒ|jƒtt|ƒj|ŒdS(N(R=R>RLtcloseRR;t__exit__(Rtargs((s7lib/python2.7/site-packages/dask/diagnostics/profile.pyRO¨s   cC`s|jƒdS(N(RH(RR((s7lib/python2.7/site-packages/dask/diagnostics/profile.pyR®scC`s|js|jƒndS(N(R>RL(RRRR-((s7lib/python2.7/site-packages/dask/diagnostics/profile.pyR.±s cC`s)|jƒr%|jjƒd|_ndS(s%Shutdown the resource tracker processN(RCR@tshutdownR?(R((s7lib/python2.7/site-packages/dask/diagnostics/profile.pyRNµs  cC`s g|_dS(N(R(R((s7lib/python2.7/site-packages/dask/diagnostics/profile.pyR½scK`s ddlm}||j|S(Ni(tplot_resources(R0RRR(RR1RR((s7lib/python2.7/site-packages/dask/diagnostics/profile.pyR2ÀscK`sddlm}|||S(sVisualize the profiling run in a bokeh plot. See also -------- dask.diagnostics.profile_visualize.visualize i(R3(R0R3(RR1R3((s7lib/python2.7/site-packages/dask/diagnostics/profile.pyR3Äs(R4R5R6RRCRHRLRRORR.RNt__del__RR2R3(((s7lib/python2.7/site-packages/dask/diagnostics/profile.pyR;js#           REcB`s5eZdZdd„Zd„Zd„Zd„ZRS(s.Background process for tracking resource usageicC`sGtj|ƒt|_||_tƒj|_tƒ\|_ |_ dS(N( RRRMtdaemonRAR tpidt parent_pidRRFt child_conn(RRA((s7lib/python2.7/site-packages/dask/diagnostics/profile.pyRÑs    cC`s:|jjs,|jjdƒ|jjƒn|jƒdS(NRQ(RFtclosedRGRNtjoin(R((s7lib/python2.7/site-packages/dask/diagnostics/profile.pyRQØs cC`sK|jgg|jjƒD]-}|j|kr|jƒdkr|^qS(Ntzombie(tparenttchildrenRUtstatus(RRUtp((s7lib/python2.7/site-packages/dask/diagnostics/profile.pyt _update_pidsÞsc C`s~tddƒ}|j|jƒ|_tƒ}g}x7trly|jjƒ}Wntk req6nX|dkrvPq6|dkrD|j |ƒ}xÕ| s«|jj ƒ r@t ƒ}d}}xU|D]M} y| j ƒj } | jƒ} Wntk rýqÅX|| 7}|| 7}qÅW|j||d|fƒt|jƒq”Wq6|dkr6|jj|ƒg}q6q6W|jjƒdS(Ntpsutils9Tracking resource usage requires `psutil` to be installedRQRDig€„.ARI(R RRVR[R RMRWRKtKeyboardInterruptR_tpollRt memory_infotrsst cpu_percentt ExceptiontappendRRARGRN( RR`RUtdatatmsgtpstticR9R:R^tmem2tcpu2((s7lib/python2.7/site-packages/dask/diagnostics/profile.pytrunâs<            (R4R5R6RRQR_Rn(((s7lib/python2.7/site-packages/dask/diagnostics/profile.pyREÏs    t CacheDatatmetrict cache_timet free_timet CacheProfilercB`s\eZdZd d d„Zd„Zd„Zd„Zd„Zd„Z d„Z d„Z RS( sùA profiler for dask execution at the scheduler cache level. Records the following information for each task: 1. Key 2. Task 3. Size metric 4. Cache entry time in seconds since the epoch 5. Cache exit time in seconds since the epoch Examples -------- >>> from operator import add, mul >>> from dask.threaded import get >>> dsk = {'x': 1, 'y': (add, 'x', 10), 'z': (mul, 'y', 2)} >>> with CacheProfiler() as prof: ... get(dsk, 'z') 22 >>> prof.results # doctest: +SKIP [CacheData('y', (add, 'x', 10), 1, 1435352238.48039, 1435352238.480655), CacheData('z', (mul, 'y', 2), 1, 1435352238.480657, 1435352238.480803)] The default is to count each task (``metric`` is 1 for all tasks). Other functions may used as a metric instead through the ``metric`` keyword. For example, the ``nbytes`` function found in ``cachey`` can be used to measure the number of bytes in the cache. >>> from cachey import nbytes # doctest: +SKIP >>> with CacheProfiler(metric=nbytes) as prof: # doctest: +SKIP ... get(dsk, 'z') The profiling results can be visualized in a bokeh plot using the ``visualize`` method. Note that this requires bokeh to be installed. >>> prof.visualize() # doctest: +SKIP You can activate the profiler globally >>> prof.register() # doctest: +SKIP If you use the profiler globally you will need to clear out old results manually. >>> prof.clear() cC`sV|jƒ|r|nd„|_|r4||_n|rI|j|_n d|_dS(NcS`sdS(Ni((R!((s7lib/python2.7/site-packages/dask/diagnostics/profile.pyt?ttcount(Rt_metrict _metric_nameR4(RRpt metric_name((s7lib/python2.7/site-packages/dask/diagnostics/profile.pyR=s  cC`s|jƒtt|ƒjƒS(N(RRRsR(R((s7lib/python2.7/site-packages/dask/diagnostics/profile.pyRGs cC`s,|jj|ƒ|js(tƒ|_ndS(N(RRt _start_timeR(RR((s7lib/python2.7/site-packages/dask/diagnostics/profile.pyRKs c C`sˆtƒ}|j|ƒ|f|j|s  Ue9