B
    ^\Y                 @   s  d dl mZmZmZ d dlmZ d dlmZ d dlmZ d dlZd dl	m
Z
mZ d dlmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZ d dlmZ d d	lmZ d dl Z d d
l!m"Z" d dl#Z#ddl$m%Z%m&Z& ddl'm(Z( ddl)m*Z* ddl$m+Z+m,Z, e j-.drddl/m0Z0 ndZ0e j-.dZ1e,e1ddZ1G dd de2Z3G dd de3Z4d-ddZ5G dd de3Z6G dd  d e3Z7G d!d" d"e3Z8G d#d$ d$e3Z9G d%d& d&e3Z:d'd( Z;d)d* Z<d+d, Z=dS ).    )print_functiondivisionabsolute_import)bisect)add)timeN)rowcolumn)ColumnDataSourcePlotDataRange1d
LinearAxis	HoverToolBoxZoomTool	ResetToolPanToolWheelZoomToolRange1dQuadTapToolOpenURLButtonSelect)	Spectral9)figure)gen   )without_property_validationBOKEH_VERSION   )
nbytes_bar)profile)
log_errorsparse_timedeltaz!distributed.dashboard.export-tool)
ExportToolz#distributed.worker.profile.intervalms)defaultc               @   s    e Zd ZdZdd Zdd ZdS )DashboardComponentaU   Base class for Dask.distributed UI dashboard components.

    This class must have two attributes, ``root`` and ``source``, and one
    method ``update``:

    *  source: a Bokeh ColumnDataSource
    *  root: a Bokeh Model
    *  update: a method that consumes the messages dictionary found in
               distributed.bokeh.messages
    c             C   s   d | _ d | _d S )N)sourceroot)self r+   ;lib/python3.7/site-packages/distributed/bokeh/components.py__init__-   s    zDashboardComponent.__init__c             C   s   dS )z? Reads from bokeh.distributed.messages and updates self.source Nr+   )r*   messagesr+   r+   r,   update1   s    zDashboardComponent.updateN)__name__
__module____qualname____doc__r-   r/   r+   r+   r+   r,   r'   !   s   
r'   c               @   s&   e Zd ZdZd	ddZedd ZdS )

TaskStreamzb Task Stream

    The start and stop time of tasks as they occur on each core of the cluster.
      20sc             K   s>   || _ t|dd}|| _d| _t|f|\| _| _dg| _dS )zO
        kwargs are applied to the bokeh.models.plots.Plot constructor
        r%   )r&   r   N)n_rectanglesr#   clear_intervallasttask_stream_figurer(   r)   task_stream_index)r*   r7   r8   kwargsr+   r+   r,   r-   ;   s    zTaskStream.__init__c          	      s   t   |d d |d d }r6d | jd kr:d S t| jd   fdd| D }d | jd< |d rttt|d |d	 }|| jkr|| j | _}||| j kr| j	j
| d S | j	|| j W d Q R X d S )
Nztask-eventsindex
rectanglesr   c                s.   i | ]&\}  fd dt tD |qS )c                s   g | ]} | qS r+   r+   ).0i)vr+   r,   
<listcomp>S   s    z0TaskStream.update.<locals>.<dictcomp>.<listcomp>)rangelen)r@   k)indr=   )rB   r,   
<dictcomp>S   s   z%TaskStream.update.<locals>.<dictcomp>startduration)r"   r;   r   itemsminmapr   r9   r8   r(   datar/   streamr7   )r*   r.   r>   mr9   r+   )rG   r=   r,   r/   I   s"    
zTaskStream.updateN)r5   r6   )r0   r1   r2   r3   r-   r   r/   r+   r+   r+   r,   r4   5   s   
r4   r6   c       
      K   s*  t | dd} ttt |  gdgdgdgdgdgdgdgd	gd
gd
d}tdd}tdd}tf ddd||ddddd	|}|j|dddddddddd
}d|_d|j_	d|j_
d|j_d|j_td d!d"}ttd#d$d%}|||t t td&d'td&d' tr"t }	|	| ||	 ||fS )(zG
    kwargs are applied to the bokeh.models.plots.Plot constructor
    r%   )r&   g?rI   Zwhitez100 msZfoor   r   g        )
rI   rJ   keynamecolorZduration_textZworkeryZworker_threadalpha)rN   )Zrange_paddingZtask_streamzTask Streamzbk-task-stream-plotZabovedatetime#    )	rR   titleidx_rangey_rangetoolbar_locationx_axis_typeZmin_border_righttoolsrT   rJ   g?rS   g333333?rU      )
r(   xrT   widthheight
fill_colorZ
line_colorZ
line_alpha
fill_alphaZ
line_widthNFfollow_mousez
            <div>
                <span style="font-size: 12px; font-weight: bold;">@name:</span>&nbsp;
                <span style="font-size: 10px; font-family: Monaco, monospace;">@duration_text</span>
            </div>
            )point_policytooltipsz/profile?key=@name)Zurl)callbackrb   )Z
dimensions)r#   r
   dictr   r   r   rectZnonselection_glyphyaxisZmajor_label_text_alphaminor_tick_line_alphaZmajor_tick_line_alphaZxgridvisibler   r   r   	add_toolsr   r   r   r   r$   Zregister_plot)
r8   r<   r(   r[   r\   r)   rk   hoverZtapZexportr+   r+   r,   r:   c   sb    



r:   c               @   s$   e Zd ZdZdd Zedd ZdS )MemoryUsagez; The memory usage across the cluster, grouped by task type c             K   s   t tg g g g g g g g dd| _tf dt t d d d|| _| j| jtddddd	dd
 | jt	 d | jt	 d t
ddd}| j| d S )N)rR   leftrightcenterrS   ZpercentZMBtext)rN   zbk-nbytes-plot)rZ   r[   r\   r]   Zoutline_line_colorr   r   rr   rs   rS   )topbottomrr   rs   rd   re   Zbelowrf   a  
                <div>
                    <span style="font-size: 14px; font-weight: bold;">Name:</span>&nbsp;
                    <span style="font-size: 10px; font-family: Monaco, monospace;">@name</span>
                </div>
                <div>
                    <span style="font-size: 14px; font-weight: bold;">Percent:</span>&nbsp;
                    <span style="font-size: 10px; font-family: Monaco, monospace;">@percent</span>
                </div>
                <div>
                    <span style="font-size: 14px; font-weight: bold;">MB:</span>&nbsp;
                    <span style="font-size: 10px; font-family: Monaco, monospace;">@MB</span>
                </div>
                )rg   rh   )r
   rj   r(   r   r   r)   Z	add_glyphr   Z
add_layoutr   r   ro   )r*   r<   rp   r+   r+   r,   r-      s     


zMemoryUsage.__init__c          	   C   s^   t  N |d }|sd S t|d }| jj| dt|d  d  | jj_	W d Q R X d S )NZprogressnbyteszMemory Use: %0.2f MBg    .A)
r"   r    r(   rN   r/   sumvaluesr)   rY   ru   )r*   r.   msgZnbr+   r+   r,   r/      s    zMemoryUsage.updateN)r0   r1   r2   r3   r-   r   r/   r+   r+   r+   r,   rq      s   'rq   c               @   s0   e Zd ZdZdd Zedd Zedd ZdS )	
Processingz Processing and distribution per core

    This shows how many tasks are actively running on each worker and how many
    tasks are enqueued for each worker and how many are in the common pool
    c             K   s   |  i i d}t|| _tdd}tf dd|dd|}|j| jdd	td d
dd d|j_d|j	_
d|j_
t }|| |t}d|_d|_|| _d S )N)
processingncoresr?   r   zProcessing and PendingrX   zbk-processing-stacks-plot)rY   r_   r[   rZ   r   rs   rv   rw   )r(   rr   rs   rS   rv   rw   Fa  
        <div>
            <span style="font-size: 14px; font-weight: bold;">Host:</span>&nbsp;
            <span style="font-size: 10px; font-family: Monaco, monospace;">@name</span>
        </div>
        <div>
            <span style="font-size: 14px; font-weight: bold;">Processing:</span>&nbsp;
            <span style="font-size: 10px; font-family: Monaco, monospace;">@processing</span>
        </div>
        rf   )processing_updater
   r(   r   r   Zquadr   Zxaxisrm   rl   rn   Zygridr   ro   selectrh   rg   r)   )r*   r<   rN   r[   Zfigrp   r+   r+   r,   r-      s     





zProcessing.__init__c          	   C   s   t   |d }|dsd S | |}| jj}t|d }t|d }|j|k r^|d |_n&|jd| | kr|jd |d  |_| jj	| W d Q R X d S )Nr}   r~   rs   r   gffffff?g?)
r"   getr   r)   r[   maxendr(   rN   r/   )r*   r.   r{   rN   r[   Z	max_rightZcoresr+   r+   r,   r/   	  s    


zProcessing.updatec          
      s   t   t| d }t|}| d fdd|D | d   fdd|D  t|}t|ttt|ddtt|d dd d	}d
g| |d< |S Q R X d S )Nr}   c                s   g | ]} | qS r+   r+   )r@   rR   )r}   r+   r,   rC      s    z0Processing.processing_update.<locals>.<listcomp>r~   c                s   g | ]} | qS r+   r+   )r@   rR   )r~   r+   r,   rC   "  s    r   r?   r   )rR   r}   rs   rv   rw   r~   gffffff?rU   )r"   sortedrE   listrD   )r{   namesndr+   )r~   r}   r,   r     s     zProcessing.processing_updateN)	r0   r1   r2   r3   r-   r   r/   staticmethodr   r+   r+   r+   r,   r|      s   r|   c               @   s$   e Zd ZdZdd Zedd ZdS )ProfilePlotz Time plots of the current resource usage on the cluster

    This is two plots, one for CPU and Memory and another for Network I/O
    c                sr   t  }t |t}|d _t j|f|\ _ _t	 fdd}t
dkr` jjd| n jd| d S )Nstatesc          	      s   t   y
|j}W n  tk
r2   |d d }Y nX y|d }W n tk
rT   d S X t j| t} jd d =  j|	d  j
j| | j
_W d Q R X d S )N1dindicesr   r   )r"   r   AttributeError
IndexErrorr!   	plot_datar   profile_intervalextendpopr(   rN   r/   selected)attroldnewr   rG   rN   )r*   r+   r,   cb<  s    
z ProfilePlot.__init__.<locals>.cbz1.0.0r   r   )r!   creater   r   r   r   plot_figurer)   r(   r   r   r   	on_change)r*   r<   staterN   r   r+   )r*   r,   r-   6  s    zProfilePlot.__init__c          	   C   sD   t  4 || _t| jt}|d| _| jj	| W d Q R X d S )Nr   )
r"   r   r!   r   r   r   r   r(   rN   r/   )r*   r   rN   r+   r+   r,   r/   R  s
    zProfilePlot.updateN)r0   r1   r2   r3   r-   r   r/   r+   r+   r+   r,   r   0  s   r   c               @   s6   e Zd ZdZd
ddZedddZeddd	ZdS )ProfileTimePlotz Time plots of the current resource usage on the cluster

    This is two plots, one for CPU and Memory and another for Network I/O
    Nc          	      st  |d k	rt |_y|jjjdd _W n tk
rF   d _Y nX t	jt
r`jd _t	jtrxj _djg_nd _dg_|_d _d _g g d_t _tjt}|d_tj|f|\__dg t fdd}td	kr jjd
| njd| t g g d_!t"f ddddddt gdd|_#j#j$ddj!d j#j%ddj!d dd dj#j&_'dj#j(_'fdd}td	kr̈j!jd
| nj!d| t)ddd_*j*+fdd t)d dd_,j,+j- t.jd! jd"_/fd#d$}j/d%| t0t1j/j*j,d&d'jj#f|_2d S )(NrQ   r   All)countr   r   Fc          	      s    d rd S t   t|tr$|}n|d d }y|d }W n tk
rP   d S X tj| t}jd d = j|	d d d< j
j| t|tr|j
j_n|j
_d d< W d Q R X d S )Nr   r   r   r   TF)r"   
isinstancer   r   r!   r   r   r   r   r   r(   rN   r/   r   r   )r   r   r   r   rG   rN   )changingr*   r+   r,   r   |  s&    

z$ProfileTimePlot.__init__.<locals>.cbz1.0.0r   r   )r   r   zActivity over timed   rV   xbox_selectr   z"xpan,xwheel_zoom,xbox_select,reset)rY   rc   r^   active_dragr\   r_   r   r   )r(   orange)r(   rS   selection_colorc          	      s   t   y jjj}W n$ tk
r:    jjd d }Y nX |r jjd t| d } jjd t| d }t||t||  _ _	nd   _ _	 j
dd W d Q R X d S )Nr   r   r   i  F)update_metadata)r"   	ts_sourcer   r   r   rN   rL   r   rI   stoptrigger_update)r   r   r   r   rI   r   )r*   r+   r,   	ts_change  s    z+ProfileTimePlot.__init__.<locals>.ts_changeResetsuccess)labelbutton_typec                  s      jS )N)r/   r   r+   )r*   r+   r,   <lambda>  s    z*ProfileTimePlot.__init__.<locals>.<lambda>Updater?   )valueoptionsc                s"   |dkrd }| _  jdd d S )Nr   F)r   )rQ   r   )r   r   r   )r*   r+   r,   	select_cb  s    z+ProfileTimePlot.__init__.<locals>.select_cbr   scale_width)sizing_mode)3weakrefrefdocZsession_contextZrequestZ	argumentsr   rQ   r   r   r   bytesdecode
task_namesserverrI   r   tsr!   r   r   r   r   r   r   r   profile_plotr(   r   r   r   r   r
   r   r   ts_plotlinecirclerl   rn   gridr   reset_buttonon_clickupdate_buttonr   r   r   r	   r   r)   )r*   r   r   r<   rN   r   r   r   r+   )r   r*   r,   r-   a  sf    







zProfileTimePlot.__init__c          	   C   s   t   || _t| jt}|d| _| jj	| |d k	r|d rdgt
|d  | _| j| j_| jrz|d | j }n|d }t| \}}|dd |D d| _| jj	| j W d Q R X d S )Nr   countsr   keysc             S   s   g | ]}|d  qS )i  r+   )r@   tr+   r+   r,   rC     s    z*ProfileTimePlot.update.<locals>.<listcomp>)r   r   )r"   r   r!   r   r   r   r   r(   rN   r/   r   r   r   r   rQ   zipr   r   )r*   r   metadatarN   r   timesr   r+   r+   r,   r/     s    
zProfileTimePlot.updateTc                s&   t j fdd} jj| d S )Nc            	   3   sv   t  f jjjjjdr0j  nd  ttj	rN gV \ 
  fdd W d Q R X d S )N)rQ   rI   r   c                  s     S )N)r/   r+   )r   profr*   r+   r,   r     s    z<ProfileTimePlot.trigger_update.<locals>.cb.<locals>.<lambda>)r"   r   get_profilerQ   rI   r   Zget_profile_metadatar   r   ZFuturer   Zadd_next_tick_callbackr+   )r*   r   )r   r   r,   r     s    z*ProfileTimePlot.trigger_update.<locals>.cb)r   	coroutiner   ZloopZadd_callback)r*   r   r   r+   )r*   r   r,   r     s    zProfileTimePlot.trigger_update)N)N)T)r0   r1   r2   r3   r-   r   r/   r   r+   r+   r+   r,   r   [  s   
kr   c               @   s2   e Zd ZdZd	ddZedd Zedd ZdS )
ProfileServerz Time plots of the current resource usage on the cluster

    This is two plots, one for CPU and Memory and another for Network I/O
    Nc          	      s  |d k	rt |_|_jjj_d _d _g g d_	t
j_tjt}|d_tj|f|\__dg t fdd}tdkrjjd| njd| tg g d	_tf d
dddddt gdd|_jjddjd jjddjd dd djj_djj_fdd}tdkrZjjd| njd| t ddd_!j!"fdd t ddd_#j#"j$ t%t&j!j#dd jjf|_'d S )!N)r   r   r   Fc          	      s    d rd S t   t|tr$|}n|d d }y|d }W n tk
rP   d S X tj| t}jd d = j|	d d d< j
j| t|tr|j
j_n|j
_d d< W d Q R X d S )Nr   r   r   r   TF)r"   r   r   r   r!   r   r   r   r   r   r(   rN   r/   r   r   )r   r   r   r   rG   rN   )r   r*   r+   r,   r     s&    

z"ProfileServer.__init__.<locals>.cbz1.0.0r   r   )r   r   zActivity over timer   rV   r   r   r   z"xpan,xwheel_zoom,xbox_select,reset)rY   rc   r^   r   r\   r_   r   r   )r(   r   )r(   rS   r   c          	      s   t   y jjj}W n$ tk
r:    jjd d }Y nX |r jjd t| d } jjd t| d }t||t||  _ _	nd   _ _	 
  W d Q R X d S )Nr   r   r   i  )r"   r   r   r   r   rN   rL   r   rI   r   r   )r   r   r   r   rI   r   )r*   r+   r,   r   /  s    z)ProfileServer.__init__.<locals>.ts_changer   r   )r   r   c                  s      jS )N)r/   r   r+   )r*   r+   r,   r   C  s    z(ProfileServer.__init__.<locals>.<lambda>r   r   )r   )(r   r   r   r   Zio_loopr!   logrI   r   r   r   r   r   r   r   r   r   r   r(   r   r   r   r   r
   r   r   r   r   r   rl   rn   r   r   r   r   r   r   r	   r   r)   )r*   r   r   r<   rN   r   r   r+   )r   r*   r,   r-     sL    




zProfileServer.__init__c          	   C   sD   t  4 || _t| jt}|d| _| jj	| W d Q R X d S )Nr   )
r"   r   r!   r   r   r   r   r(   rN   r/   )r*   r   rN   r+   r+   r,   r/   L  s
    zProfileServer.updatec             C   s   t j| j| j| jd| _t | jt}|d| _	| j
j| dd | jD }ttdtd| j}| jj||d d S )N)rI   r   r   c             S   s   g | ]\}}|d  qS )i  r+   )r@   r   _r+   r+   r,   rC   Z  s    z0ProfileServer.trigger_update.<locals>.<listcomp>r   r   )r   r   )r!   r   r   rI   r   r   r   r   r   r   r(   rN   r/   r   toolzZpluckr   )r*   rN   r   r   r+   r+   r,   r   T  s    zProfileServer.trigger_update)N)r0   r1   r2   r3   r-   r   r/   r   r+   r+   r+   r,   r     s   
Ur   c                s,   t | |  fdd| t| | dS )a   Add periodic callback to doc in a way that avoids reference cycles

    If we instead use ``doc.add_periodic_callback(component.update, 100)`` then
    the component stays in memory as a reference cycle because its method is
    still around.  This way we avoid that and let things clean up a bit more
    nicely.

    TODO: we still have reference cycles.  Docs seem to be referred to by their
    add_periodic_callback methods.
    c                  s   t  S )N)r/   r+   )r   r+   r,   r   l  s    z'add_periodic_callback.<locals>.<lambda>N)r   r   add_periodic_callback_attach)r   	componentZintervalr+   )r   r,   r   _  s    
r   c             C   s   |  }|d k	r|   d S )N)r/   )r   compr+   r+   r,   r/   p  s    r/   c             C   s"   t | dst | _| j| d S )N
components)hasattrsetr   r   )r   r   r+   r+   r,   r   v  s    
r   )r6   )>Z
__future__r   r   r   r   operatorr   r   r   Zbokeh.layoutsr   r	   Zbokeh.modelsr
   r   r   r   r   r   r   r   r   r   r   r   r   r   r   Zbokeh.palettesr   Zbokeh.plottingr   ZdaskZtornador   r   Zutilsr   r   Zdiagnostics.progress_streamr    rX   r!   r"   r#   Zconfigr   Zexport_toolr$   r   objectr'   r4   r:   rq   r|   r   r   r   r   r/   r   r+   r+   r+   r,   <module>   s@   D.
J6M+ n