B
    t\pf              	   @   s  d dl mZmZmZ d dlmZ d dlZd dlZd dlZd dl	m
Z
mZmZ d dlmZmZmZmZmZmZmZmZmZ d dlmZmZ d dlmZ d dlmZ d d	lmZ d d
l m!Z!m"Z" ddl#m$Z$m%Z%m&Z&m'Z' ddl(m)Z) ddl*m+Z+m,Z, ddl-m.Z. ddl/m0Z0 ddl1m2Z2 ddl*m3Z3m4Z4m5Z5m6Z6 e7e8Z9e:ej;<ej;=e>ddZ?e?@ ZAW dQ R X d dlBmCZCmDZD eCeDej;<ej;=e>ddZEeej;<ej;=e>dZFdddddgiZGG dd de$ZHG d d! d!e$ZIG d"d# d#e$ZJG d$d% d%e$ZKG d&d' d'e$ZLG d(d) d)e$ZMG d*d+ d+e$ZNd d,lOmPZP d d-lQmRZR d.d/ ZSd0d1 ZTd2d3 ZUd4d5 ZVd6d7 ZWd8d9 ZXG d:d; d;e)ZYdS )<    )print_functiondivisionabsolute_import)partialN)rowcolumn	widgetbox)	ColumnDataSourceDataRange1d	HoverToolBoxZoomTool	ResetToolPanToolWheelZoomToolNumeralTickFormatterSelect)	DataTableTableColumn)figure)RdBu)Theme)mergepartition_all   )DashboardComponentProfileTimePlotProfileServeradd_periodic_callback)BokehServer)	transposewithout_property_validation   )WINDOWS)color_of)time)
log_errors	key_splitformat_bytesformat_timeZ	templatesz	base.html)EnvironmentFileSystemLoader)loaderz
theme.yamlZpagesmainsystemprofilecrossfilterc               @   s$   e Zd ZdZdd Zedd ZdS )
StateTablez Currently running tasks c                s`   || _ ddddddg}tdd |D | _d	d |D  t| j fd
d|D dd}|| _d S )NStored	ExecutingReadyWaitingConnectionsServingc             S   s   i | ]
}g |qS  r7   ).0namer7   r7   7lib/python3.7/site-packages/distributed/bokeh/worker.py
<dictcomp>1   s    z'StateTable.__init__.<locals>.<dictcomp>c             S   s   i | ]}t ||d |qS ))Zfieldtitle)r   )r8   r9   r7   r7   r:   r;   3   s   c                s   g | ]} | qS r7   r7   )r8   n)columnsr7   r:   
<listcomp>7   s    z'StateTable.__init__.<locals>.<listcomp>F   )sourcer>   height)workerr	   rA   r   root)selfrC   namestabler7   )r>   r:   __init__-   s    zStateTable.__init__c          	   C   sv   t  f | j}t|jgdt|j|jf gt|jgt|jgt|jgt|j	gd}| j
j| W d Q R X d S )Nz%d / %d)r1   r2   r3   r4   r5   r6   )r%   rC   lendata	executingncoresZreadyZwaiting_for_datain_flight_workers_commsrA   update)rE   wdr7   r7   r:   rO   <   s    



zStateTable.updateN)__name__
__module____qualname____doc__rH   r    rO   r7   r7   r7   r:   r0   *   s   r0   c               @   s"   e Zd ZdddZedd ZdS )CommunicatingStream,  c       	      K   s  t   || _ddddddddd	d
g
}tdd |D | _tdd |D | _tdd}tdd}tf dd|||dd|}|j| jddddddd |j| jddddddd tddd}|	|t
 tddtdd || _d| _d| _t | _W d Q R X d S )Nstartstopmiddledurationwhoyhoveralpha	bandwidthtotalc             S   s   i | ]
}g |qS r7   r7   )r8   r9   r7   r7   r:   r;   P   s    z0CommunicatingStream.__init__.<locals>.<dictcomp>c             S   s   i | ]
}g |qS r7   r7   )r8   r9   r7   r7   r:   r;   Q   s    r   )range_paddingzPeer Communicationsdatetime )r<   x_axis_typex_rangey_rangerB   toolsg?red)rA   xr]   widthrB   colorr_   bluefollow_mousez@hover)point_policytooltipsrk   )
dimensions)r%   rC   r	   incomingoutgoingr
   r   rectr   	add_toolsr   r   r   rD   last_incominglast_outgoingdictr\   )	rE   rC   rB   kwargsrF   rf   rg   figr^   r7   r7   r:   rH   J   s6    

zCommunicatingStream.__init__c                s  t   | jj| jj| j }fddtd|d D | jj| _| jj | jj| j } fddtd|d D  | jj| _xr | j	g| j
ggD ]X\}}x|D ]}d|kr|d= |d= |d |d pd	 }tt|d
 dd}||d< y| j|d  |d< W n< tk
rD   t| j| j|d < | j|d  |d< Y nX dt|d t|d t|d |d  f |d< xdD ]}|| d ||< q~W qW |rt|}t|jd rt|d |jd d d kr|j| q|j|dd qW W d Q R X d S )Nc                s   g | ]} |    qS r7   )copy)r8   i)rs   r7   r:   r?   u   s    z.CommunicatingStream.update.<locals>.<listcomp>r   c                s   g | ]} |    qS r7   )r{   )r8   r|   )rr   r7   r:   r?   z   s    Z
compressedkeysra   r[   g      ?g    eͽAg333333?r_   r\   r]   z%s / %s = %s/sr^   )rZ   r[   rX   rY   i  rY   rX   i'  )rollover)r%   rC   outgoing_transfer_logoutgoing_countrw   rangeincoming_transfer_logincoming_countrv   rr   rs   maxminr\   KeyErrorrI   r'   r(   r   rJ   rO   stream)rE   r=   ZmsgsrA   msgr`   Zbwkr7   )rr   rs   r:   rO   p   sF    






 zCommunicatingStream.updateN)rW   )rR   rS   rT   rH   r    rO   r7   r7   r7   r:   rV   I   s   
&rV   c               @   s    e Zd Zdd Zedd ZdS )CommunicatingTimeSeriesc          	   K   s   || _ tg g g d| _tdddd}tf ddd|jd	 gd
d|d|}|j| jdddd |j| jdddd |t t	ddt
dd || _d S )N)rj   inoutendi N  r   )followfollow_intervalrb   zCommunication Historyrc   gg      ?   rd   )r<   re   rg   rB   rh   rf   rj   r   ri   )rA   rj   r]   rl   r   rm   rk   )rq   )rC   r	   rA   r
   r   Ztotal_out_connectionslineru   r   r   r   rD   )rE   rC   ry   rf   rz   r7   r7   r:   rH      s    z CommunicatingTimeSeries.__init__c          	   C   sH   t  8 | jt d gt| jjgt| jjgdd W d Q R X d S )Ni  )rj   r   r   i'  )r%   rA   r   r$   rI   rC   rN   rM   )rE   r7   r7   r:   rO      s
    zCommunicatingTimeSeries.updateN)rR   rS   rT   rH   r    rO   r7   r7   r7   r:   r      s   r   c               @   s    e Zd Zdd Zedd ZdS )ExecutingTimeSeriesc          	   K   s   || _ tg g d| _tdddd}tf ddd|jd	 gd
d|d|}|j| jddd |t t	ddt
dd || _d S )N)rj   r]   r   i N  r   )r   r   rb   zExecuting Historyrc   gg?r   rd   )r<   re   rg   rB   rh   rf   rj   r]   )rA   rj   r]   rk   )rq   )rC   r	   rA   r
   r   rL   r   ru   r   r   r   rD   )rE   rC   ry   rf   rz   r7   r7   r:   rH      s    zExecutingTimeSeries.__init__c          	   C   s<   t  , | jt d gt| jjgdd W d Q R X d S )Ni  )rj   r]   )r%   rA   r   r$   rI   rC   rK   )rE   r7   r7   r:   rO      s    zExecutingTimeSeries.updateN)rR   rS   rT   rH   r    rO   r7   r7   r7   r:   r      s   r   c               @   s<   e Zd Zdd Zedd Zdd Zedd Zd	d
 ZdS )CrossFilterc             K   s  t  r || _ddddddg}ddd	g}td
dgddgddgd
dgddgddgddgddgd
dgd
dgd
| _tdd|d| _| jd| j tdd|d| _| jd| j tdddg| d| _	| j	d| j tdddg| d| _
| j
d| j d|krd|d i}ni }t| j| j| j	| j
gfddi|| _d | _d | _|| _t| j| jf | jf|| _| j| _W d Q R X d S )!Nnbytesr[   r`   countrX   rY   zinout-colorz
type-colorz	key-colorr   r!   g{Gz?g{Gz?intstrrm   ri   addZinc)
r   r[   r`   r   typezinout-colorz
type-colorkeyrX   rY   zX-Axis)r<   valueZoptionsr   zY-AxisZSizeNoneZColorZblacksizing_moderk      r   )r%   rC   r	   rA   r   rj   Z	on_changeupdate_figurer]   sizerl   r   Zcontrolrw   rv   ry   r   create_figurelayoutrD   )rE   rC   ry   
quantitiesZcolorskwr7   r7   r:   rH      sJ    


zCrossFilter.__init__c          	      sf  t  T | jj| jj| j }t|d}fddtd|D | jj| _| jj | jj| j	 }t|d} fddtd|D  | jj| _	g }x0 D ](}|d r| 
|}d|d< || qW x0D ](}|d r| 
|}d	|d< || qW |rXt|}t| jjd
 rHt|d | jjd
 d d krH| jj| n| jj|dd W d Q R X d S )Ni  c                s   g | ]} |    qS r7   )r{   )r8   r|   )rs   r7   r:   r?     s    z&CrossFilter.update.<locals>.<listcomp>r   c                s   g | ]} |    qS r7   )r{   )r8   r|   )rr   r7   r:   r?     s    r}   ri   zinout-colorrm   rY   rX   r~   
   )r   )r%   rC   r   r   rw   r   r   r   r   rv   process_msgappendr   rI   rA   rJ   rO   r   )rE   r=   r   r   rQ   r7   )rr   rs   r:   rO     s8    








"zCrossFilter.updatec          
   K   s   t   tf ddd|}| jj}|dkr0d}|j| j| jj| jj| jjdddd | jj|j	_
| jj|j_
|t t t t  |S Q R X d S )Nrd   )r<   rh   r   r   r   g      ?)rA   rj   r]   rl   r   r_   Zhover_alpha)r%   r   r   r   ZcirclerA   rj   r]   rl   xaxis
axis_labelyaxisru   r   r   r   r   )rE   ry   rz   r   r7   r7   r:   r   4  s     
zCrossFilter.create_figurec          	   C   s0   t    | jf | j}|| jjd< W d Q R X d S )Nr   )r%   r   ry   r   Zchildren)rE   attroldnewrz   r7   r7   r:   r   K  s    zCrossFilter.update_figurec                s   yx fdd}t  d |d}| jj|tj}t|} d  d  d t d |t||t| d  d	 d

}|S  t	k
r } zt
|  W d d }~X Y nX d S )Nc                s    d  | dS )Nr}   r   )get)r   )r   r7   r:   funcS  s    z%CrossFilter.process_msg.<locals>.funcr}   )r   ra   r[   r`   rX   rY   )
r   r[   r`   r   r   z
type-colorr   z	key-colorrX   rY   )r   rC   typesr   objectrR   r&   rI   r#   	ExceptionloggerZ	exception)rE   r   r   Zmain_keytypZkeynamerQ   er7   )r   r:   r   Q  s&    

zCrossFilter.process_msgN)	rR   rS   rT   rH   r    rO   r   r   r   r7   r7   r7   r:   r      s
   5%r   c               @   s*   e Zd Zd	ddZdd Zedd ZdS )
SystemMonitorr   c       	      K   s  || _ |jj}d| _tdd |D | _| jj|   t	dddd}d}t
f dd	|||d
|| _| jj| jddd d| jj_t
f dd	|||d
|| _| jj| jddd d| jj_t
f dd	|||d|| _| jj| jdddd | jj| jdddd d| jj_tdd| jjd _tdd| jjd _| j| j| jg}tst
f dd	|||d|| _| jj| jddd || j d|krd|d i}ni }tsd| jj_d| jj_d| jj_d| jj_t||| _| j j  d S )Nr   c             S   s   i | ]
}g |qS r7   r7   )r8   r9   r7   r7   r:   r;   p  s    z*SystemMonitor.__init__.<locals>.<dictcomp>r   i N  )r   r   rb   zreset,xpan,xwheel_zoomZCPUrc   )r<   re   rB   rh   rf   r$   cpu)rA   rj   r]   Z
PercentageZMemoryZmemoryZBytesZ	Bandwidth)r<   re   rB   rf   rh   Z
read_bytesri   )rA   rj   r]   rl   Zwrite_bytesrm   zBytes / secondz0.0b)formatzNumber of File Descriptorsnum_fdsr   )rC   monitorr   lastr	   rA   rJ   rO   get_datar
   r   r   r   r   r   Zmemr`   r   	formatterr"   r   r   rg   rX   r   rD   )	rE   rC   rB   ry   rF   rf   rh   Zplotsr   r7   r7   r:   rH   k  sV    







zSystemMonitor.__init__c             C   s8   | j jj| jd}dd |d D |d< | j jj| _|S )N)rX   c             S   s   g | ]}|d  qS )i  r7   )r8   rj   r7   r7   r:   r?     s    z*SystemMonitor.get_data.<locals>.<listcomp>r$   )rC   r   Zrange_queryr   r   )rE   rQ   r7   r7   r:   r     s    zSystemMonitor.get_datac          	   C   s(   t   | j|  d W d Q R X d S )Ni  )r%   rA   r   r   )rE   r7   r7   r:   rO     s    zSystemMonitor.updateN)r   )rR   rS   rT   rH   r   r    rO   r7   r7   r7   r:   r   j  s   
:r   c               @   s2   e Zd ZdddZdd Zdd Zedd	 Zd
S )Countersstretch_bothc                s   || _ i | _i | _i | _i | _| _| j jrHx| j jD ]}| | q6W x| j jD ]}| 	| qRW t
| j| j  fddt D  t dkrt d| _n$tfddtd D di| _d S )Nc                s   g | ]} | qS r7   r7   )r8   r   )figuresr7   r:   r?     s    z%Counters.__init__.<locals>.<listcomp>   )r   c                s   g | ]}t |d  iqS )r   )r   )r8   Zpair)r   r7   r:   r?     s   r!   r   )servercounter_figurescounter_sourcesdigest_figuresdigest_sourcesr   digestsadd_digest_figurecountersadd_counter_figurer   sortedrI   r   rD   r   )rE   r   r   ry   r9   r7   )r   r   r:   rH     s$    zCounters.__init__c             C   s  t   t| jj| j}dd t|D }i }|drDd|d< tf |dd| jd|}d	|j	_
d	|j_
|d
s|drtdd|jd _xLt|D ]@}dd||  |  }|j|| dd|tt|d |  d qW tjd |j_d |j_|| j|< || j|< |S Q R X d S )Nc             S   s   i | ]}t g g d |qS ))rj   r]   )r	   )r8   r|   r7   r7   r:   r;     s   z.Counters.add_digest_figure.<locals>.<dictcomp>r[   rc   re   rd   r   )r<   rh   rB   r   Fr`   bytesz0.0b)r   r   g333333?rj   r]      )rA   rj   r]   r_   rl      )r%   rI   r   r   	intervalsr   endswithr   r   r   visibleygridr   r   r   r   r   r   mathpimajor_label_orientationtoolbarlogor   r   )rE   r9   r=   sourcesry   rz   r|   r_   r7   r7   r:   r     s,    




zCounters.add_digest_figurec             C   s   t   t| jj| j}dd t|D }t|dd| jtt	t
| jj| jd d}d|j_xpt|D ]d}dd	| |  }|j|| d
d|ddtt|d |  d tddd}|| tjd |j_qjW d |j_|| j|< || j|< |S Q R X d S )Nc             S   s    i | ]}t g g g g d |qS ))rj   r]   zy-centercounts)r	   )r8   r|   r7   r7   r:   r;     s   z/Counters.add_counter_figure.<locals>.<dictcomp>rd   r   r   )r<   rh   rB   r   rf   Fg      ?g?rj   zy-centerr]   g333333?r   )rA   rj   r]   rk   rB   r_   rl   rn   z@x : @counts)ro   rp   r   )r%   rI   r   r   r   r   r   r   r   mapr   
componentsr   r   rt   r   r   r   ru   r   r   r   r   r   r   r   r   )rE   r9   r=   r   rz   r|   rk   r^   r7   r7   r:   r     s*     


zCounters.add_counter_figurec       
   	      s  t   x| j D ]\}}| jj| }i  xft|jD ]X\}   r: d\}}|dd  }|	drv|d9 }| j
| | j||d q:W d|| f |j_qW x| j D ]\}}| jj| }i  xt|jD ]\}  rlt }|jd |j|   fdd	|D }fd
d	|D }dd	 |D }	ttt|}|||	|d | j| | j  d|| f |j_ttt||j_qW qW W d Q R X d S )Nd   r   r[   i  )rj   r]   z%s: %dr   c                s   g | ]} | qS r7   r7   )r8   rj   )rQ   r7   r:   r?     s    z#Counters.update.<locals>.<listcomp>c                s   g | ]} | qS r7   r7   )r8   c)factorr7   r:   r?     s    c             S   s   g | ]}|d  qS )r!   r7   )r8   r]   r7   r7   r:   r?     s    )rj   r]   zy-centerr   )r%   r   itemsr   r   	enumerater   r   Z	histogramr   r   rJ   rO   r<   textr   r   r   r   listr   r   r   rf   Zfactors)
rE   r9   rz   Zdigestr|   ZysZxscounterr   Z	y_centersr7   )rQ   r   r:   rO     s8    

 zCounters.updateN)r   )rR   rS   rT   rH   r   r   r    rO   r7   r7   r7   r:   r     s   
r   )FunctionHandler)Applicationc          
   C   s   t   t| }t| dd}t| dd}t| dd}|jj}||j_||j_d|_t||d t||d t||d t||d |	t
|j|j|j|jdd td|_d|jd< |j| t|_W d Q R X d S )Nscale_width)r   zDask Worker Internal Monitorr   zsimple.htmlr,   active_page)r%   r0   r   r   rV   rD   rf   r<   r   add_rootr   envget_templatetemplatetemplate_variablesrO   BOKEH_THEMEtheme)rC   extradoc
statetableZexecuting_tsZcommunicating_tsZcommunicating_streamZxrr7   r7   r:   main_doc(  s0    


r   c          	   C   s   t  p t| }t| }d|_t||d t||d |t|j|j t	d|_
d|jd< |j| t|_W d Q R X d S )NzDask Worker Cross-filteri  zsimple.htmlr/   r   )r%   r0   r   r<   r   r   r   rD   r   r   r   r   rO   r   r   )rC   r   r   r   r/   r7   r7   r:   crossfilter_docE  s    
r   c          	   C   sh   t  X t| dd}d|_t||d ||j td|_d|j	d< |j	
| t|_W d Q R X d S )Nr   )r   zDask Worker Monitori  zsimple.htmlr-   r   )r%   r   r<   r   r   rD   r   r   r   r   rO   r   r   )rC   r   r   Zsysmonr7   r7   r:   systemmonitor_docU  s    
r   c          	   C   sh   t  X d|_t| dd}t||d ||j td|_d|j	d< |j	
| t|_W d Q R X d S )NzDask Worker Countersr   )r   i  zsimple.htmlr   r   )r%   r<   r   r   r   rD   r   r   r   r   rO   r   r   )r   r   r   r   r7   r7   r:   counters_docb  s    
r   c          	   C   sd   t  T d|_t| dd}|  ||j td|_d|j	d< |j	
| t|_W d Q R X d S )NzDask Worker Profiler   )r   zsimple.htmlr.   r   )r%   r<   r   trigger_updater   rD   r   r   r   r   rO   r   r   )r   r   r   r.   r7   r7   r:   profile_doco  s    
r   c          	   C   s\   t  L d|_t| d|d}||j td|_|j	| t
|_|  W d Q R X d S )NzDask: Profile of Event Loopr   )r   r   zsimple.html)r%   r<   r   r   rD   r   r   r   r   rO   r   r   r   )r   r   r   Zprofr7   r7   r:   profile_server_doc|  s    r   c                   s>   e Zd ZdddZedd Zedd Z fd	d
Z  ZS )BokehWorkerNrd   c             K   s   || _ || _|pd | jd< |p d}|d}|rB|dsBd| }|| _d|i}|t ttt	t
||}ttt	t||}ttt	t||}ttt	t||}	ttt	t||}
ttt	t||}||	|||
|d| _|p|j| _d | _d S )Nprefixrd   /)z/mainz	/countersz/crossfilterz/systemz/profilez/profile-server)rC   Zserver_kwargsrstrip
startswithr   rO   r   r   r   r   r   r   r   r   r   r   ZappsZloopr   )rE   rC   Zio_loopr   ry   r   r,   r/   Zsystemmonitorr   r.   Zprofile_serverr7   r7   r:   rH     s0    


zBokehWorker.__init__c             C   s   t d| jitS )Nr   )r   r   r   )rE   r7   r7   r:   r     s    zBokehWorker.extrac             C   s   | j S )N)rC   )rE   r7   r7   r:   	my_server  s    zBokehWorker.my_serverc                sD   t t j|| ddlm}  fdd|D } jjd| d S )Nr   )routesc                s.   g | ]&\}} j d  | | j jdfqS )r   )r   r   )r   r   r   )r8   Zurlcls)rE   r7   r:   r?     s   z&BokehWorker.listen.<locals>.<listcomp>z.*)superr   listenZworker_htmlr   r   Z_tornadoZadd_handlers)rE   argsry   r   Zhandlers)	__class__)rE   r:   r    s
    
zBokehWorker.listen)Nrd   )	rR   rS   rT   rH   propertyr   r   r  __classcell__r7   r7   )r  r:   r     s   
r   )ZZ
__future__r   r   r   	functoolsr   Zloggingr   osZbokeh.layoutsr   r   r   Zbokeh.modelsr	   r
   r   r   r   r   r   r   r   Zbokeh.models.widgetsr   r   Zbokeh.plottingr   Zbokeh.palettesr   Zbokeh.themesr   Ztoolzr   r   r   r   r   r   r   Zcorer   Zutilsr   r    Zcompatibilityr"   Zdiagnostics.progress_streamr#   Zmetricsr$   r%   r&   r'   r(   Z	getLoggerrR   r   openpathjoindirname__file__freadZtemplate_sourceZjinja2r)   r*   r   r   r   r0   rV   r   r   r   r   r   Z#bokeh.application.handlers.functionr   Zbokeh.applicationr   r   r   r   r   r   r   r   r7   r7   r7   r:   <module>   sT   ,
 V Gs