B
    ^\s—  ã               @   s¢  d dl mZmZmZ d dlZd dlmZ d dlmZ d dl	m
Z
 d dlZd dlmZ d dlZd dlZd dlZd dlZd dlmZ d dlZd dlZd dlZd dlZd dlZd dlmZ d d	lmZ d dlZd dlZd dlZd dl Z d dl!Z!d dl"Z"d dl#Z$d
dl%m&Z&m'Z'm(Z(m)Z) yd dl*Z*W n e+k
r6   dZ*Y nX d dl,Z,d dl,m-Z- d dl.Z.d dl/Z/d dl/m0Z0 d dl1m2Z2 yd dl1m3Z3 W n e+k
rž   dZ3Y nX d
dl%m4Z4m5Z5m6Z6m7Z7m8Z8 d
dl9mZ yd dl:m;Z; W n e+k
rö   e <¡ Z;Y nX e =e>¡ Z?Z@dZAdd„ ZBeBƒ ZCdd„ ZDdd„ ZEdd„ ZFe.jGdd„ ƒZHdÈd d!„ZIdÉd#d$„ZJd%d&„ ZKed'd(„ ƒZLe0jMd)d*„ ƒZNe0jMdÊd,d-„ƒZOd.d/„ ZPG d0d1„ d1eQƒZRed2d3„ ƒZSed4d5„ ƒZTd6d7„ ZUd8d9„ ZVe Wd:¡ZXd;d<„ ZYyd d=lmZZZ W n e+k
r    d>ZZY nX eZd?ƒeYƒZYe5rd@dA„ Z[ndBdA„ Z[edËdCdD„ƒZ\dÌdFdG„Z]e.jGdHdI„ ƒZ^e$j_ `¡  dJdK„ ZadÍdMdN„ZbejcdOkr~eddPƒ ndQdR„ ZedSdT„ ZfdÎdUdV„ZgdWdX„ ZhdYdZ„ Zid[d\„ Zjd]d^„ ZkdÏd_d`„Zldadb„ ZmdÐdcdd„ZnedÑdfdg„ƒZodhdi„ Zpdjdk„ Zqdldm„ Zresednƒrejtfdodp„Zun(d>gZvevfdqdr„Zwevfdsdp„Zue xew¡ dteu_ydÒdudv„Zzdwdx„ Z{G dydz„ dzeQƒZ|d{d|„ Z}d}d~dd€dd‚dƒd„d…d†d
d
d‡œZ~dˆd‰„ e~ ¡ D ƒZ~e~ €dŠd‰„ e~ ¡ D ƒ¡ e~ €d‹d‰„ e~ ¡ D ƒ¡ dŒd„ Zd
dŽddd‘d’d“d”œZ‚d
d‘d’d“dŽddd•œZƒeƒ €d–d‰„ eƒ ¡ D ƒ¡ e‚ €eƒ¡ e‚ €d—d‰„ e‚ ¡ D ƒ¡ dÓd™dš„Z„d›dœ„ Z…e6rde†e‡eˆffddž„Z‰ne†e‡ffdŸdž„Z‰dÔd d¡„ZŠed¢d£„ ƒZ‹ejŒfd¤d¥„Zd¦d§„ ZŽG d¨d©„ d©ejƒZdªd«„ Z‘d¬ej’krBe/jcd  d­krBd>Z“d®ej’krd dl”Z”d d¯l•m–Z– e”j—j˜ ™¡ oeše”j—j˜ ›¡ e–ƒZ“e“sBd dlœZœd dlZ/eœ že/jŸjœ  ¡ ¡ d°d±„ Z¡eZr\eZd}ƒe¡ƒZ¡d²d³d´dµd¶d·d¸d¹dºd»d¼d½d¾d¿dÀdÁdÂdÃgZ¢e.jGe¢fdÄdÅ„ƒZ£dÆdÇ„ Z¤dS )Õé    )Úprint_functionÚdivisionÚabsolute_importN)Údeque)Úcontextmanager)Ú	timedelta)Úmd5)ÚNumber)Úsleep)Úimport_moduleé   )Úcache_from_sourceÚ
getargspecÚinvalidate_cachesÚreload)Úistask)Úgen)ÚIOLoop)Ú
PollIOLoop)ÚQueueÚPY3ÚPY2Úget_thread_identityÚunicode)Útime)Úthread_stateZ__no_default__c              C   s^   t rVtj d¡sVdtjkrVtj d¡} t 	| ¡}dg}dtj
krJ| d¡ | |¡ nt}|S )NÚwinZPyPyz)distributed.worker.multiprocessing-methodÚdistributedÚpkg_resources)r   ÚsysÚplatformÚ
startswithÚversionÚdaskÚconfigÚgetÚmultiprocessingZget_contextÚmodulesÚappendZset_forkserver_preload)ÚmethodZctxZpreload© r*   ú0lib/python3.7/site-packages/distributed/utils.pyÚ_initialize_mp_context?   s    


r,   c             C   s:   xt | dƒr| j} qW y| jS  tk
r4   t| ƒS X dS )zGet the name of a function.ÚfuncN)Úhasattrr-   Ú__name__ÚAttributeErrorÚstr)r-   r*   r*   r+   ÚfuncnameQ   s    
r2   c             C   sZ   xTy|t | ƒjkrdS W n tk
r.   P Y nX y
| j} W q tk
rP   P Y qX qW dS )zE
    Whether the function takes an argument with the given name.
    TF)r   ÚargsÚ	TypeErrorÚ__wrapped__r0   )r-   Úargnamer*   r*   r+   Úhas_arg[   s    
r7   c               C   s    t dk	rt  t j¡d S dS dS )z;
    Get the maximum number of open files per process.
    Nr   i   )ÚresourceZ	getrlimitZRLIMIT_NOFILEr*   r*   r*   r+   Úget_fileno_limitm   s    r9   c          
   C   sz   t   |t j¡}z\y| | |f¡ | ¡ d }|S  tk
rf } zt d| ||f t¡ |S d }~X Y nX W d | ¡  X d S )Nr   zKCouldn't detect a suitable IP address for reaching %r, defaulting to %r: %s)	ÚsocketZ
SOCK_DGRAMZconnectÚgetsocknameÚEnvironmentErrorÚwarningsÚwarnÚRuntimeWarningÚclose)ÚhostÚportÚfamilyÚdefaultZsockZipÚer*   r*   r+   Ú_get_ipy   s    rF   ú8.8.8.8éP   c             C   s   t | |tjddS )z¦
    Get the local IP address through which the *host* is reachable.

    *host* defaults to a well-known Internet host (one of Google's public
    DNS servers).
    z	127.0.0.1)rC   rD   )rF   r:   ÚAF_INET)rA   rB   r*   r*   r+   Úget_ipŒ   s    rJ   ú2001:4860:4860::8888c             C   s   t | |tjddS )z-
    The same as get_ip(), but for IPv6.
    z::1)rC   rD   )rF   r:   ÚAF_INET6)rA   rB   r*   r*   r+   Úget_ipv6–   s    rM   c             C   sB   ddl }x&| ¡ |  D ]}|jtjkr|jS qW td| f ƒ‚dS )zÓ
    Get the local IPv4 address of a network interface.

    KeyError is raised if the interface doesn't exist.
    ValueError is raised if the interface does no have an IPv4 address
    associated with it.
    r   Nz)interface %r doesn't have an IPv4 address)ÚpsutilZnet_if_addrsrC   r:   rI   ZaddressÚ
ValueError)ZifnamerN   Úinfor*   r*   r+   Úget_ip_interface   s
    
rQ   c           
   g   s2   y
d V  W n" | k
r, } zW d d }~X Y nX d S )Nr*   )Ú
exceptionsrE   r*   r*   r+   Úignoring¬   s    
rS   c          	   g   sR   t j| Ž }g }x4| ¡ sBt|Ž  | ¡ V }| |¡ W dQ R X qW t  |¡‚dS )zÅ Process list of coroutines, ignoring certain exceptions

    >>> coroutines = [cor(...) for ...]  # doctest: +SKIP
    >>> x = yield ignore_exceptions(coroutines, TypeError)  # doctest: +SKIP
    N)r   ÚWaitIteratorÚdonerS   Únextr(   ÚReturn)Z
coroutinesrR   Zwait_iteratorÚresultsÚresultr*   r*   r+   Úignore_exceptions´   s    



rZ   r*   c             #   s~   t j| Ž ‰dd„ | D ƒ}xVˆ ¡ snyˆ ¡ V }W n0 tk
r`   t j‡ ‡fdd„ƒ}|ƒ  ‚ Y nX ||ˆj< qW t  |¡‚dS )a"   Wait on many tasks at the same time

    Err once any of the tasks err.

    See https://github.com/tornadoweb/tornado/issues/1546

    Parameters
    ----------
    args: futures to wait for
    quiet_exceptions: tuple, Exception
        Exception types to avoid logging if they fail
    c             S   s   g | ]}d ‘qS )Nr*   )Ú.0Ú_r*   r*   r+   ú
<listcomp>Ó   s    zAll.<locals>.<listcomp>c           	   3   s8   x2t ˆjƒD ]$} y
| V  W q ˆ k
r.   Y qX qW dS )zè Watch unfinished tasks

                Otherwise if they err they get logged in a way that is hard to
                control.  They need some other task to watch them so that they
                are not orphaned
                N)ÚlistZ_unfinished)Útask)Úquiet_exceptionsÚtasksr*   r+   ÚquietØ   s
    
zAll.<locals>.quietN)r   rT   rU   rV   Ú	ExceptionÚ	coroutineZcurrent_indexrW   )r3   r`   rX   rY   rb   r*   )r`   ra   r+   ÚAllÄ   s    

re   c          	      sä   t r4t| t ƒrt| ddƒs,t| dƒr4| jjr4tdƒ‚ˆ dd¡‰t 	¡ ‰t
ƒ ‰dg‰dg‰tj‡ ‡‡‡‡‡‡‡fdd„ƒ}|  |¡ ˆdk	r¨ˆ ˆ¡sÀt d	ˆf ¡‚nxˆ ¡ s¾ˆ d
¡ qªW ˆd rØtjˆd Ž  nˆd S dS )z;
    Run coroutine in loop running in separate thread.
    Z_closingFÚasyncio_loopzIOLoop is closedZcallback_timeoutNc           
   3   sœ   z†yRˆt ƒ krtdƒ‚tjV  dt_ˆˆ ˆŽ} ˆd k	rHt tˆd| ¡} | V ˆd< W n. tk
r‚ } zt	 
¡ ˆd< W d d }~X Y nX W d dt_ˆ ¡  X d S )Nz)sync() called from thread of running loopT)Úsecondsr   F)r   ÚRuntimeErrorr   Zmomentr   ÚasynchronousZwith_timeoutr   rc   r   Úexc_infoÚset)ZfutureÚexc)r3   rE   Úerrorr-   ÚkwargsÚmain_tidrY   Útimeoutr*   r+   Úfü   s    

"zsync.<locals>.fztimed out after %s s.é
   r   )r   Ú
isinstanceÚgetattrr.   rf   Z_closedrh   ÚpopÚ	threadingÚEventr   r   rd   Úadd_callbackÚwaitÚTimeoutErrorZis_setÚsixZreraise)Úloopr-   r3   rn   rq   r*   )r3   rE   rm   r-   rn   ro   rY   rp   r+   Úsyncì   s$     


r}   c               @   sp   e Zd ZdZe ¡ Ze ¡ Z	ddd„Z
dd„ Zdd	„ Zddd„Zdd„ Zdd„ Zdd„ Zdd„ Zedd„ ƒZdS )Ú
LoopRunnerag  
    A helper to start and stop an IO loop in a controlled way.
    Several loop runners can associate safely to the same IO loop.

    Parameters
    ----------
    loop: IOLoop (optional)
        If given, this loop will be re-used, otherwise an appropriate one
        will be looked up or created.
    asynchronous: boolean (optional, default False)
        If false (the default), the loop is meant to run in a separate
        thread and will be started if necessary.
        If true, the loop is meant to run in the thread this
        object is instantiated from, and will not be started automatically.
    NFc          	   C   sp   t  ¡ }|d kr,|r|| _nt ƒ | _d| _n|| _d| _|| _d | _d| _| j | j 	| jd¡ W d Q R X d S )NTF)r   N)
r   ÚcurrentÚ_loopZ_should_close_loopÚ_asynchronousÚ_loop_threadÚ_startedÚ_lockÚ
_all_loopsÚ
setdefault)Úselfr|   ri   r   r*   r*   r+   Ú__init__.  s    zLoopRunner.__init__c          	   C   s   | j  |  ¡  W dQ R X dS )z£
        Start the IO loop if required.  The loop is run in a dedicated
        thread.

        If the loop is already running, this method does nothing.
        N)r„   Ú_start_unlocked)r‡   r*   r*   r+   ÚstartA  s    zLoopRunner.startc                sr  | j r
t‚| j| j \}}| js0|d k	s0|dkrN|d |f| j| j< d| _ d S | jd ks\t‚|dksht‚t ¡ ‰t ¡ ‰ d g‰d g‰‡‡fdd„‰| jf‡ ‡‡fdd„	}tj|dd	}d|_	| 
¡  ˆjd
d d| _ ˆd }||k	r>ˆ  d¡ tˆd tƒs(tˆd tƒs tdˆd ƒ‚ˆd ‚|d d f| j| j< n0ˆd d ksTtˆƒ‚|| _|d | f| j| j< d S )Nr   r   Tc                  s   t  ¡ ˆ d< ˆ ¡  d S )Nr   )rv   Zcurrent_threadrk   r*   )Ú	in_threadÚloop_evtr*   r+   Úloop_cb\  s    z+LoopRunner._start_unlocked.<locals>.loop_cbc          
      sV   |   ˆ¡ z<y|  ¡  W n* tk
rB } z|ˆd< W d d }~X Y nX W d ˆ  ¡  X d S )Nr   )rx   rŠ   rc   rk   )r|   rE   )Údone_evtr   Ú	start_excr*   r+   Úrun_loop`  s    
z,LoopRunner._start_unlocked.<locals>.run_loopzIO loop)ÚtargetÚnamerr   )rp   é   znot an exception)rƒ   ÚAssertionErrorr…   r€   r   r‚   rv   rw   ÚThreadÚdaemonrŠ   ry   rs   rh   rc   r4   )r‡   ÚcountÚreal_runnerr   ZthreadZactual_threadr*   )rŽ   r‹   r   rŒ   r   r+   r‰   K  s<    
	

zLoopRunner._start_unlockedrr   c          	   C   s    | j  |  |¡ W dQ R X dS )zv
        Stop and close the loop if it was created by us.
        Otherwise, just mark this object "stopped".
        N)r„   Ú_stop_unlocked)r‡   rp   r*   r*   r+   Ústop~  s    zLoopRunner.stopc             C   sj   | j s
d S d| _ | j| j \}}|dkr>|d |f| j| j< n(|dksJt‚| j| j= |d k	rf| |¡ d S )NFr   )rƒ   r…   r€   r”   Ú
_real_stop)r‡   rp   r—   r˜   r*   r*   r+   r™   †  s    
zLoopRunner._stop_unlockedc          	   C   sf   | j d k	st‚| j d k	rbz@| j | jj¡ | j j|d ttƒ | j ¡  W d Q R X W d d | _ X d S )N)rp   )	r‚   r”   r€   rx   rš   ÚjoinrS   ÚKeyErrorr@   )r‡   rp   r*   r*   r+   r›   •  s    

zLoopRunner._real_stopc             C   s   | j S )zP
        Return True between start() and stop() calls, False otherwise.
        )rƒ   )r‡   r*   r*   r+   Ú
is_started   s    zLoopRunner.is_startedc             O   sF   | j rt| j|f|ž|ŽS |  ¡  zt| j|f|ž|ŽS |  ¡  X dS )z‚
        Convenience helper: start the loop if needed,
        run sync(func, *args, **kwargs), then stop the loop again.
        N)rƒ   r}   r|   rŠ   rš   )r‡   r-   r3   rn   r*   r*   r+   Úrun_sync¦  s    zLoopRunner.run_syncc             C   s   | j S )N)r€   )r‡   r*   r*   r+   r|   ´  s    zLoopRunner.loop)NF)rr   )r/   Ú
__module__Ú__qualname__Ú__doc__ÚweakrefÚWeakKeyDictionaryr…   rv   ZLockr„   rˆ   rŠ   r‰   rš   r™   r›   rž   rŸ   Úpropertyr|   r*   r*   r*   r+   r~     s   

3
r~   c              k   s´   i }x4| D ],}yt t|ƒ||< W q
 tk
r4   Y q
X q
W x |  ¡ D ]\}}tt||ƒ qDW z
d V  W d xD| D ]<}y|| }W n tk
rœ   tt|ƒ Y qnX tt||ƒ qnW X d S )N)rt   r   r0   ÚitemsÚsetattrr   Údelattr)rn   ÚoldÚkÚvr*   r*   r+   Úset_thread_state¹  s     



r¬   c          	   c   sZ   t j t ¡ | ¡}t|dƒ}| |¡ W d Q R X z
|V  W d t j |¡rTt  |¡ X d S )NÚw)	ÚosÚpathrœ   ÚtempfileZ
gettempdirÚopenÚwriteÚexistsÚremove)ÚfilenameÚtextÚfnrq   r*   r*   r+   Útmp_textÏ  s    
r¸   c             C   s   x|   ¡ s|  ¡  qW d S )N)ÚemptyZ
get_nowait)Úqr*   r*   r+   Úclear_queueÜ  s    
r»   c              C   s,   dt jkrdS ddlm}  t| ƒ ddƒdk	S )zX Determine if we're running within an IPython kernel

    >>> is_kernel()
    False
    ÚIPythonFr   )Úget_ipythonZkernelN)r   r'   r¼   r½   rt   )r½   r*   r*   r+   Ú	is_kernelá  s    
r¾   z[a-f]+c             C   s  t | ƒtkr|  ¡ } t | ƒtkr(| d } yÐ|  d¡}|d d  ¡ s^|d  d¡d  d¡}n|d }xF|dd… D ]6}| ¡ r¨t|ƒdkršt 	|¡dk	s¨|d| 7 }qtP qtW t|ƒdkrÊt
 	d	|¡rÊd
S |d dkrò| d¡ ¡ d  d¡d }|S W n tk
r   dS X dS )a5  
    >>> key_split('x')
    'x'
    >>> key_split('x-1')
    'x'
    >>> key_split('x-1-2-3')
    'x'
    >>> key_split(('x-2', 1))
    'x'
    >>> key_split("('x-2', 1)")
    'x'
    >>> key_split("('x', 1)")
    'x'
    >>> key_split('hello-world-1')
    'hello-world'
    >>> key_split(b'hello-world-1')
    'hello-world'
    >>> key_split('ae05086432ca935f6eba409a8ecd4896')
    'data'
    >>> key_split('<module.submodule.myclass object at 0xdaf372')
    'myclass'
    >>> key_split(None)
    'Other'
    >>> key_split('x-abcdefab')  # ignores hex
    'x'
    r   ú-ú,z'("r   Né   é    z[a-f0-9]{32}Údataú<z<>Ú.éÿÿÿÿÚOther)ÚtypeÚbytesÚdecodeÚtupleÚsplitÚisalphaÚstripÚlenÚhex_patternÚmatchÚrerc   )ÚsZwordsrY   Zwordr*   r*   r+   Ú	key_splitò  s*    
rÔ   )Ú	lru_cacheFi † c             C   s¨   t | ƒ}|tkr| d S |tkrŒ| d dkrB|  dd¡d  d¡S t| ƒdkr^t d| ¡r^dS | d d	kr†|  d
¡ ¡ d  d¡d S | S n|tkr t	|  
¡ ƒS dS dS )aï  A more fine-grained version of key_split

        >>> key_split_group('x')
        'x'
        >>> key_split_group('x-1')
        'x-1'
        >>> key_split_group('x-1-2-3')
        'x-1-2-3'
        >>> key_split_group(('x-2', 1))
        'x-2'
        >>> key_split_group("('x-2', 1)")
        'x-2'
        >>> key_split_group('hello-world-1')
        'hello-world-1'
        >>> key_split_group(b'hello-world-1')
        'hello-world-1'
        >>> key_split_group('ae05086432ca935f6eba409a8ecd4896')
        'data'
        >>> key_split_group('<module.submodule.myclass object at 0xdaf372')
        'myclass'
        >>> key_split_group(None)
        'Other'
        >>> key_split_group('x-abcdefab')  # ignores hex
        'x-abcdefab'
        r   ú(rÀ   r   z()"'rÂ   z[a-f0-9]{32}rÃ   rÄ   z<>rÅ   rÆ   rÇ   N)rÈ   rË   r1   rÌ   rÎ   rÏ   rÒ   rÑ   rÉ   Úkey_split_grouprÊ   )ÚxÚtypr*   r*   r+   r×   0  s    r×   c             C   sœ   t | ƒ}|tkr| d S |tks(|tkr”| d dkrJ|  dd¡d  d¡S t| ƒdkrft d| ¡rfdS | d d	krŽ|  d
¡ ¡ d  d¡d S | S ndS dS )aï  A more fine-grained version of key_split

        >>> key_split_group('x')
        'x'
        >>> key_split_group('x-1')
        'x-1'
        >>> key_split_group('x-1-2-3')
        'x-1-2-3'
        >>> key_split_group(('x-2', 1))
        'x-2'
        >>> key_split_group("('x-2', 1)")
        'x-2'
        >>> key_split_group('hello-world-1')
        'hello-world-1'
        >>> key_split_group(b'hello-world-1')
        'hello-world-1'
        >>> key_split_group('ae05086432ca935f6eba409a8ecd4896')
        'data'
        >>> key_split_group('<module.submodule.myclass object at 0xdaf372')
        'myclass'
        >>> key_split_group(None)
        'Other'
        >>> key_split_group('x-abcdefab')  # ignores hex
        'x-abcdefab'
        r   rÖ   rÀ   r   z()"'rÂ   z[a-f0-9]{32}rÃ   rÄ   z<>rÅ   rÆ   rÇ   N)	rÈ   rË   r1   r   rÌ   rÎ   rÏ   rÒ   rÑ   )rØ   rÙ   r*   r*   r+   r×   [  s    c             c   s’   ddl m} y
d V  W nv |tjfk
r2   ‚ Y n\ tk
rŒ } z>yt |¡ W n tk
rd   Y nX | rzdd l} |  	¡  ‚ W d d }~X Y nX d S )Nr   )ÚCommClosedErrorr   )
ZcommrÚ   r   rW   rc   ÚloggerZ	exceptionr4   ÚpdbZ	set_trace)rÜ   rÚ   rE   r*   r*   r+   Ú
log_errors…  s    
rÝ   r   c             C   sV   t | tƒrtt|  ¡ ƒ} d}t |¡}x*|jD ] }t |tjƒr.|j}| 	| ¡ q.W |S )z
    Force all existing loggers below *root* to the given level at least
    (or keep the existing level if less verbose).
    N)
rs   r1   rt   ÚloggingÚupperÚ	getLoggerÚhandlersZStreamHandlerÚlevelZsetLevel)râ   Úrootr©   rÛ   Úhandlerr*   r*   r+   Úsilence_logging—  s    

rå   c             C   sr   t jt jg}x\|D ]T}yt  | d|t j¡}W n( t jk
rT } z|}W dd}~X Y qX |d d d S qW |‚dS )zÌ Ensure that address is an IP address

    Examples
    --------
    >>> ensure_ip('localhost')
    '127.0.0.1'
    >>> ensure_ip('123.123.123.123')  # pass through IP addresses
    '123.123.123.123'
    iÒ  Nr   é   )r:   rI   rL   ZgetaddrinfoÚSOCK_STREAMZgaierror)ZhostnameZfamiliesZfamrX   rE   rl   r*   r*   r+   Ú	ensure_ip©  s    
rè   c                 sl   t  ¡ \} }‰ tj dd¡tj dd¡tj dd¡tj dd¡g}x$ˆ rft‡ fdd	„|D ƒƒrfˆ j‰ qDW ˆ S )
Nr   ZworkerZ	schedulerÚtornadozgen.pyZ
concurrentZfuturesc             3   s   | ]}|ˆ j jjkV  qd S )N)Útb_frameÚf_codeÚco_filename)r[   Úb)Úexc_tracebackr*   r+   ú	<genexpr>Ì  s   z get_traceback.<locals>.<genexpr>)r   rj   r®   r¯   rœ   ÚanyÚtb_next)Úexc_typeÚ	exc_valueZbadr*   )rî   r+   Úget_tracebackÆ  s    

rô   é'  c          	   C   sb   t t| ƒƒ|krZyt| ƒdt| ƒd|… ƒS  tk
rV   tdt| ƒt| ƒd|… ƒS X n| S dS )z1 Truncate exception to be about a certain length zLong error messageN)rÏ   r1   rÈ   rc   )rE   Únr*   r*   r+   Útruncate_exceptionÒ  s    r÷   )é   zÂdef queue_to_iterator(q):
        while True:
            result = q.get()
            if isinstance(result, StopIteration):
                return result.value
            yield result
        c             c   s&   x |   ¡ }t|tƒr|‚|V  qW d S )N)r%   rs   ÚStopIteration)rº   rY   r*   r*   r+   Úqueue_to_iteratorë  s
    
rú   c             C   s   x| D ]}|  |¡ qW d S )N)Zput)Úseqrº   Úitemr*   r*   r+   Ú_dump_to_queueó  s    
rý   c             C   s.   t |d}tjt| |fd}d|_| ¡  |S )N)Úmaxsize)r‘   r3   T)r   rv   r•   rý   r–   rŠ   )rû   rþ   rº   Útr*   r*   r+   Úiterator_to_queueø  s
    
r   c             C   s(   t | ƒ}|tks|tkr| S t| ƒS dS )z Convert an object to a string.

    Examples
    --------

    >>> tokey(b'x')
    'x'
    >>> tokey('x')
    'x'
    >>> tokey(1)
    '1'
    N)rÈ   r   rÉ   r1   )ÚorÙ   r*   r*   r+   Útokey  s    r  c             C   s,   t | ƒ}|tk	r(|tk	r(td|| f ƒ‚dS )z,Validate a key as received on a stream.
    z"Unexpected key type %s (value: %r)N)rÈ   r   rÉ   r4   )rª   rÙ   r*   r*   r+   Úvalidate_key  s    r  c             C   s@   t | ƒp>t| ƒtkr"ttt| ƒƒp>t| ƒtko>ttt|  ¡ ƒƒS )z! Possibly contains a nested task )r   rÈ   r^   rð   ÚmapÚ_maybe_complexÚdictÚvalues)r_   r*   r*   r+   r    s    r  c                s¬   t | ƒtkr ‡ ‡fdd„| D ƒS t | ƒtkrD‡ ‡fdd„|  ¡ D ƒS t| ƒrv| d ft‡ ‡fdd„| dd … D ƒƒ S y| ˆ ksˆ| ˆkrt| ƒS W n tk
r¦   Y nX | S )	Nc                s   g | ]}t |ˆ ˆƒ‘qS r*   )Úconvert)r[   r«   )ÚdskÚextra_valuesr*   r+   r]   (  s    zconvert.<locals>.<listcomp>c                s   i | ]\}}t |ˆ ˆƒ|“qS r*   )r  )r[   rª   r«   )r	  r
  r*   r+   ú
<dictcomp>*  s    zconvert.<locals>.<dictcomp>r   c             3   s   | ]}t |ˆ ˆƒV  qd S )N)r  )r[   rØ   )r	  r
  r*   r+   rï   ,  s    zconvert.<locals>.<genexpr>r   )rÈ   r^   r  r¦   r   rË   r  r4   )r_   r	  r
  r*   )r	  r
  r+   r  &  s    *r  c                s   ‡ ‡fdd„ˆ   ¡ D ƒS )Nc                s"   i | ]\}}t |ˆ ˆƒt|ƒ“qS r*   )r  r  )r[   rª   r«   )r	  r
  r*   r+   r  6  s    zstr_graph.<locals>.<dictcomp>)r¦   )r	  r
  r*   )r	  r
  r+   Ú	str_graph5  s    r  c             C   s   |   ¡ dkrdS d}xv|  |¡}|s(dS || }y0| |¡}|  |   ¡ t|ƒ|  t|ƒ ¡ dS  tk
rt   Y nX |t|ƒ d… }qW dS )aœ   Seek current file to next byte after a delimiter bytestring

    This seeks the file to the next byte following the delimiter.  It does
    not return anything.  Use ``file.tell()`` to see location afterwards.

    Parameters
    ----------
    file: a file
    delimiter: bytes
        a delimiter like ``b'
'`` or message sentinel
    blocksize: int
        Number of bytes to read from the file at once.
    r   Nó    )ÚtellÚreadÚindexÚseekrÏ   rO   )ÚfileÚ	delimiterZ	blocksizeZlastr   ZfullÚir*   r*   r+   Úseek_delimiter9  s    

"r  c             C   st   |r\|   |¡ t| |dƒ |  ¡ }||| 8 }|   || ¡ t| |dƒ |  ¡ }|}|| }|   |¡ |  |¡}|S )a%   Read a block of bytes from a file

    Parameters
    ----------
    f: file
        File-like object supporting seek, read, tell, etc..
    offset: int
        Byte offset to start read
    length: int
        Number of bytes to read
    delimiter: bytes (optional)
        Ensure reading starts and stops at delimiter bytestring

    If using the ``delimiter=`` keyword argument we ensure that the read
    starts and stops at delimiter boundaries that follow the locations
    ``offset`` and ``offset + length``.  If ``offset`` is zero then we
    start at zero.  The bytestring returned WILL include the
    terminating delimiter string.

    Examples
    --------

    >>> from io import BytesIO  # doctest: +SKIP
    >>> f = BytesIO(b'Alice, 100\nBob, 200\nCharlie, 300')  # doctest: +SKIP
    >>> read_block(f, 0, 13)  # doctest: +SKIP
    b'Alice, 100\nBo'

    >>> read_block(f, 0, 13, delimiter=b'\n')  # doctest: +SKIP
    b'Alice, 100\nBob, 200\n'

    >>> read_block(f, 10, 10, delimiter=b'\n')  # doctest: +SKIP
    b'Bob, 200\nCharlie, 300'
    i   )r  r  r  r  )rq   ÚoffsetÚlengthr  rŠ   ÚendrÉ   r*   r*   r+   Ú
read_blockZ  s    "


r  Ú c             c   s‚   d|   d¡ } t | ¡\}}t |¡ t |¡ |V  tj |¡r~tj |¡rZt	 
|¡ n$yt |¡ W n tk
r|   Y nX d S )NrÅ   )Úlstripr°   Zmkstempr®   r@   r´   r¯   r³   ÚisdirÚshutilZrmtreeÚOSError)Ú	extensionZhandlerµ   r*   r*   r+   ÚtmpfileŽ  s    

r   c             C   sb   t | tƒr| S t | tƒr |  ¡ S t | tƒs8tr@t | tƒr@t| ƒS t| dƒrR|  ¡ S t	d|  ƒ‚dS )zs Turn string or bytes to bytes

    >>> ensure_bytes('123')
    b'123'
    >>> ensure_bytes(b'123')
    b'123'
    Úencodez<Object %s is neither a bytes object nor has an encode methodN)
rs   rÉ   Ú
memoryviewÚtobytesÚ	bytearrayr   Úbufferr.   r!  r4   )rÓ   r*   r*   r+   Úensure_bytes¡  s    


r&  c             C   sN   t |ƒ}d}g }x8|D ]0}| | | | }t|dƒ\}}| t|ƒ¡ qW |S )a  

    >>> divide_n_among_bins(12, [1, 1])
    [6, 6]
    >>> divide_n_among_bins(12, [1, 2])
    [4, 8]
    >>> divide_n_among_bins(12, [1, 2, 1])
    [3, 6, 3]
    >>> divide_n_among_bins(11, [1, 2, 1])
    [2, 6, 3]
    >>> divide_n_among_bins(11, [.1, .2, .1])
    [2, 6, 3]
    g        r   )ÚsumÚdivmodr(   Úint)rö   ZbinsZtotalZaccÚoutrí   Znowr*   r*   r+   Údivide_n_among_binsµ  s    
r+  c             C   s   t | ƒ} t| ƒt| ƒ S )N)r^   r'  rÏ   )rû   r*   r*   r+   ÚmeanÍ  s    r,  Úis_finalizingc             C   s   | ƒ S )Nr*   )r-  r*   r*   r+   Úshutting_downÓ  s    r.  c             C   s   d| d< d S )NTr   r*   )Úlr*   r*   r+   Ú_at_shutdownÙ  s    r0  c             C   s   | d S )Nr   r*   )r/  r*   r*   r+   r.  Ü  s    a  
    Whether the interpreter is currently shutting down.
    For use in finalizers, __del__ methods, and similar; it is advised
    to early bind this function rather than look it up when calling it,
    since at shutdown module globals may be cleared.
    c             C   s@   t   t jt j¡}| | df¡ | d¡ | ¡ d }| ¡  |S )zš Return a probably-open port

    There is a chance that this port will be taken by the operating system soon
    after returning from this function.
    r   r   )r:   rI   rç   ZbindZlistenr;   r@   )rA   rÓ   rB   r*   r*   r+   Ú	open_portê  s    
r1  c          	   C   sd  t j | ¡\}}t j |¡\}}g }d}|dkrH|tjkr>|}| |¡ |dkrvt| ƒ}ttƒ t  	|¡ W dQ R X |dkrÞ| tjkr–tj 
d| ¡ |dkrÌddl}| | ¡}	x,|	D ]}
| |
j¡ q¶W n|dkrÞ| |¡ g }|sôt d|¡ nltƒ  |dk	rtj 
d|¡ z4x.|D ]&}t d	||¡ | tt|ƒƒ¡ qW W d|dk	r^tj 	|¡ X |S )
z, Loads modules for a file (.py, .zip, .egg) N)z.pyz.py)z.eggz.zipz.pyzr   z.egg)z.zipz.pyzzFound nothing to import from %szReload module %s from %s file)r®   r¯   rÌ   Úsplitextr   r(   r   rS   r  r´   Úinsertr   Zfind_distributionsZproject_namerÛ   Zwarningr   rP   r   r   )r¯   Z	directoryrµ   r’   ZextZnames_to_importZtmp_python_pathZ
cache_filer   ZpkgsÚpkgZloadedr*   r*   r+   Úimport_fileù  sF    









r5  c               @   s,   e Zd ZdZdZdd„ Zdd„ Zdd„ Zd	S )
Ú
itemgetterzˆA picklable itemgetter.

    Examples
    --------
    >>> data = [0, 1, 2]
    >>> get_1 = itemgetter(1)
    >>> get_1(data)
    1
    )r  c             C   s
   || _ d S )N)r  )r‡   r  r*   r*   r+   rˆ   0  s    zitemgetter.__init__c             C   s
   || j  S )N)r  )r‡   rØ   r*   r*   r+   Ú__call__3  s    zitemgetter.__call__c             C   s   t | jffS )N)r6  r  )r‡   r*   r*   r+   Ú
__reduce__6  s    zitemgetter.__reduce__N)r/   r    r¡   r¢   Ú	__slots__rˆ   r7  r8  r*   r*   r*   r+   r6  $  s
   	r6  c             C   sl   | dkrd| d  S | dkr(d| d  S | dkr<d| d  S | dkrPd| d  S | d	krdd
| d  S d|  S )a*   Format bytes as text

    >>> format_bytes(1)
    '1 B'
    >>> format_bytes(1234)
    '1.23 kB'
    >>> format_bytes(12345678)
    '12.35 MB'
    >>> format_bytes(1234567890)
    '1.23 GB'
    >>> format_bytes(1234567890000)
    '1.23 TB'
    >>> format_bytes(1234567890000000)
    '1.23 PB'
    g  4&õkCz%0.2f PBg   ¢”mBz%0.2f TBg    eÍÍAz%0.2f GBg    €„.Az%0.2f MBg     @@z%0.2f kBiè  z%d Br*   )rö   r*   r*   r+   Úformat_bytes:  s    r:  iè  i@B i Êš;l    J)£l     Iú5 i   i   i   @l        l           )ZkBZMBZGBZTBZPBZKiBZMiBZGiBZTiBZPiBÚBr  c             C   s   i | ]\}}||  ¡ “qS r*   )Úlower)r[   rª   r«   r*   r*   r+   r  e  s    r  c             C   s&   i | ]\}}|rd |kr||d “qS )r  r   r*   )r[   rª   r«   r*   r*   r+   r  f  s    c             C   s*   i | ]"\}}|rd |kr||dd… “qS )r  NrÆ   r*   )r[   rª   r«   r*   r*   r+   r  g  s    c             C   s   |   dd¡} | d  ¡ s d|  } x*tt| ƒd ddƒD ]}| |  ¡ s6P q6W |d }| d|… }| |d… }t|ƒ}t| ¡  }|| }t|ƒS )a†   Parse byte string to numbers

    >>> parse_bytes('100')
    100
    >>> parse_bytes('100 MB')
    100000000
    >>> parse_bytes('100M')
    100000000
    >>> parse_bytes('5kB')
    5000
    >>> parse_bytes('5.4 kB')
    5400
    >>> parse_bytes('1kiB')
    1024
    >>> parse_bytes('1e6')
    1000000
    >>> parse_bytes('1e6 kB')
    1000000000
    >>> parse_bytes('MB')
    1000000
    ú r  r   Ú1r   rÆ   N)	ÚreplaceÚisdigitÚrangerÏ   rÍ   ÚfloatÚ
byte_sizesr<  r)  )rÓ   r  r  ÚprefixÚsuffixrö   Ú
multiplierrY   r*   r*   r+   Úparse_bytesj  s    rG  gü©ñÒMbP?gíµ ÷Æ°>g•Ö&è.>é<   i  i€Q )rÓ   ZmsÚusÚnsÚmÚhÚd)ÚsecondZminuteZhourZdayZmillisecondZmicrosecondZ
nanosecondc             C   s   i | ]\}}||d  “qS )rÓ   r*   )r[   rª   r«   r*   r*   r+   r  §  s    c             C   s   i | ]\}}||  ¡ “qS r*   )rß   )r[   rª   r«   r*   r*   r+   r  ©  s    rg   c       	      C   sÈ   t | tƒr|  ¡ S t | tƒr$t| ƒ} |  dd¡} | d  ¡ sDd|  } x*tt| ƒd ddƒD ]}| |  	¡ sZP qZW |d }| d|… }| |d… p’|}t
|ƒ}t| ¡  }|| }t|ƒ|krÄt|ƒ}|S )a   Parse timedelta string to number of seconds

    Examples
    --------
    >>> parse_timedelta('3s')
    3
    >>> parse_timedelta('3.5 seconds')
    3.5
    >>> parse_timedelta('300ms')
    0.3
    >>> parse_timedelta(timedelta(seconds=3))  # also supports timedeltas
    3
    r=  r  r   r>  r   rÆ   N)rs   r   Ztotal_secondsr	   r1   r?  r@  rA  rÏ   rÍ   rB  Útimedelta_sizesr<  r)  )	rÓ   rD   r  r  rD  rE  rö   rF  rY   r*   r*   r+   Úparse_timedelta¬  s&    

rP  c                s    dd„ |D ƒ}t dd„ | D ƒƒ} t dd„ tt|Ž | ƒD ƒƒ}ddt| ƒ  | ‰ ˆ t | ƒ }dd	 d
d„ |D ƒ¡ }d ‡ fdd„|D ƒ¡}d |||||g¡S )zöFormats an ascii table for given columns and rows.

    Parameters
    ----------
    columns : list
        The column names
    rows : list of tuples
        The rows in the table. Each tuple must be the same length as
        ``columns``.
    c             S   s   g | ]}t d d„ |D ƒƒ‘qS )c             s   s   | ]}t |ƒV  qd S )N)r1   )r[   r  r*   r*   r+   rï   ß  s    z(asciitable.<locals>.<listcomp>.<genexpr>)rË   )r[   Úrr*   r*   r+   r]   ß  s    zasciitable.<locals>.<listcomp>c             s   s   | ]}t |ƒV  qd S )N)r1   )r[   r  r*   r*   r+   rï   à  s    zasciitable.<locals>.<genexpr>c             s   s*   | ]"\}}t t tt|ƒƒt|ƒƒV  qd S )N)Úmaxr  rÏ   )r[   rØ   Úcr*   r*   r+   rï   á  s   ú|z	 %%-%ds |z+%s+ú+c             s   s   | ]}d |d  V  qdS )r¿   é   Nr*   )r[   r­   r*   r*   r+   rï   å  s    Ú
c             3   s   | ]}ˆ | V  qd S )Nr*   )r[   rQ  )Úrow_templater*   r+   rï   æ  s    )rË   ÚziprÏ   rœ   )ÚcolumnsZrowsZwidthsÚheaderZbarrÃ   r*   )rX  r+   Ú
asciitableÔ  s    r\  c             C   sL   t | |ƒrt| ƒS t | tƒrB| jdkr,| jS t tj| j| j¡S n| j	S dS )z* Number of bytes of a frame or memoryview N)
rs   rÏ   r"  ÚshapeÚitemsizeÚ	functoolsÚreduceÚoperatorÚmulÚnbytes)ÚframeÚ_bytes_liker*   r*   r+   rc  ë  s    



rc  c             C   s6   t | |ƒrt| ƒS y| jS  tk
r0   t| ƒS X dS )z* Number of bytes of a frame or memoryview N)rs   rÏ   rc  r0   )rd  re  r*   r*   r+   rc  ø  s    
c             C   s,   t jdkrt j | |¡S t j | ||¡S dS )z‡
    Wrapper around tornado.IOLoop.PeriodicCallback, for compatibility
    with removal of the `io_loop` parameter in Tornado 5.0.
    )r“   N)ré   Úversion_infoZioloopÚPeriodicCallback)ÚcallbackZcallback_timeZio_loopr*   r*   r+   rg    s    
rg  c             c   s2   t ƒ }d V  t ƒ }|| | kr.td||| ƒ d S )NzTIME WARNING)r   Úprint)Zdurationr¶   rŠ   r  r*   r*   r+   Ú	time_warn  s
    rj  c          
   C   sz   xt j | ¡stdƒ qW xZtdƒD ]N}y(t| ƒ}||ƒ}W dQ R X |rN|S W n ttfk
rh   Y nX tdƒ q$W dS )zB Reads a JSON file from disk that may be being written as we read g{®Gáz„?rr   Ngš™™™™™¹?)r®   r¯   r³   r
   rA  r±   rO   r   )r·   Úloadr  rq   Zcfgr*   r*   r+   Újson_load_robust  s    
rl  c             C   s0   | dkrd|  S | dkr$d| d  S d| d  S )zÊ format integers as time

    >>> format_time(1)
    '1.00 s'
    >>> format_time(0.001234)
    '1.23 ms'
    >>> format_time(0.00012345)
    '123.45 us'
    >>> format_time(123.456)
    '123.46 s'
    r   z%.2f sgü©ñÒMbP?z%.2f msg     @@z%.2f usg    €„.Ar*   )rö   r*   r*   r+   Úformat_time&  s
    rm  c                   sD   e Zd ZdZe ¡ Z‡ fdd„Zdd„ Zdd„ Z	e
dd	„ ƒZ‡  ZS )
ÚDequeHandlerz5 A logging.Handler that records records into a deque c                s:   |  dd¡}t|d| _tt| ƒj||Ž | j | ¡ d S )Nrö   i'  )Úmaxlen)ru   r   Úsuperrn  rˆ   Ú
_instancesÚadd)r‡   r3   rn   rö   )Ú	__class__r*   r+   rˆ   =  s    zDequeHandler.__init__c             C   s   | j  |¡ d S )N)r   r(   )r‡   Úrecordr*   r*   r+   ÚemitC  s    zDequeHandler.emitc             C   s   | j  ¡  dS )z)
        Clear internal storage.
        N)r   Úclear)r‡   r*   r*   r+   rv  F  s    zDequeHandler.clearc             C   s    xt | jƒD ]}| ¡  qW dS )zG
        Clear the internal storage of all live DequeHandlers.
        N)r^   rq  rv  )ÚclsZinstr*   r*   r+   Úclear_all_instancesL  s    z DequeHandler.clear_all_instances)r/   r    r¡   r¢   r£   ÚWeakSetrq  rˆ   ru  rv  Úclassmethodrx  Ú__classcell__r*   r*   )rs  r+   rn  9  s   rn  c              C   s:   x4t jjj ¡ D ]"} xt  | ¡jD ]}| ¡  q"W qW dS )zo Python 2's logger's locks don't survive a fork event

    https://github.com/dask/distributed/issues/1491
    N)rÞ   ZLoggerZmanagerZ
loggerDictÚkeysrà   rá   Z
createLock)r’   rä   r*   r*   r+   Úreset_logger_locksU  s    r}  Úasyncior“   Znotebook)ÚNotebookAppc             C   s8   t r|t | ¡jkS t | ¡r$| j} |t | ¡jkS d S )N)	r   ÚinspectZ	signatureZ
parametersr   Úis_coroutine_functionr5   r   r3   )r-   Úkeywordr*   r*   r+   Úhas_keywordr  s
    
rƒ  z#440154z#471669z#472A79z#433C84z#3C4D8Az#355D8Cz#2E6C8Ez#287A8Ez#23898Dz#1E978Az#20A585z#2EB27Cz#45BF6Fz#64CB5Dz#88D547z#AFDC2Ez#D7E219z#FDE724c             C   s6   t t| ƒ ¡ ƒ}t| ¡ d d… dƒ}||t|ƒ  S )NrÁ   é   )r   r1   r!  r)  Z	hexdigestrÏ   )rØ   ÚpaletterL  rö   r*   r*   r+   Úcolor_of‡  s    r†  c             C   s*   t  | ¡rdS tjdkr&t | ¡r&dS dS )NT)rø   r“   F)r   r  r   rf  r€  Úiscoroutinefunction)rq   r*   r*   r+   r‡  Ž  s
    
r‡  )rG   rH   )rK   rH   )r*   )F)r   )rõ   )r   )r*   )N)r  )r  )rg   )N)¥Z
__future__r   r   r   ÚatexitÚcollectionsr   Ú
contextlibr   Zdatetimer   r_  Zhashlibr   r€  ZjsonrÞ   r&   Znumbersr	   ra  r®   rÒ   r  r:   r   r
   Ú	importlibr   r   r°   rv   r=   r£   r{   Ztblib.pickling_supportZtblibZcompatibilityr   r   r   r   r8   ÚImportErrorr#   r   Ztoolzré   r   Ztornado.ioloopr   r   r   r   r   r   r   ZmetricsZdask.contextr   Zlocalrà   r/   rÛ   Z_loggerZ
no_defaultr,   Z
mp_contextr2   r7   r9   ZmemoizerF   rJ   rM   rQ   rS   rd   rZ   re   r}   Úobjectr~   r¬   r¸   r»   r¾   ÚcompilerÐ   rÔ   rÕ   r×   rÝ   rå   rè   Zpickling_supportZinstallrô   r÷   rf  Úexecrú   rý   r   r  r  r  r  r  r  r  r   r&  r+  r,  r.   r-  r.  Z_shutting_downr0  Úregisterr¢   r1  r5  r6  r:  rC  r¦   ÚupdaterG  rO  Ztds2rP  r\  rÉ   r$  r%  rc  rg  rj  rk  rl  rm  ZHandlerrn  r}  r'   Zjupyter_event_loop_initializedZ	traitletsZnotebook.notebookappr  r$   ZApplicationZinitializedrs   Úinstancer~  Ztornado.platform.asyncioZset_event_loop_policyr    ZAnyThreadEventLoopPolicyrƒ  r…  r†  r‡  r*   r*   r*   r+   Ú<module>   sf  





'.  
5
+*




	
!
4

++

(
	
