
\c           @` s	  d  d l  m Z m Z m Z d  d l Z d  d l m Z d  d l m Z d  d l	 m
 Z
 d  d l Z d  d l m Z d  d l Z d  d l Z d  d l Z d  d l Z d  d l m Z d  d l Z d  d l Z d  d l Z d  d l Z d  d l Z d  d l m Z d  d	 l m Z d  d l Z d  d l Z d  d l Z d  d l  Z  d  d l! Z! d  d l" Z" d  d l# Z$ d
 d l% m& Z& m' Z' m( Z( m) Z) y d  d l* Z* Wn e+ k
 rd Z* n Xd  d l- Z- d  d l- m. Z. d  d l/ Z/ d  d l0 Z0 d  d l0 m1 Z1 d  d l2 m3 Z3 y d  d l2 m4 Z4 Wn e+ k
 r1d Z4 n Xd
 d l% m5 Z5 m6 Z6 m7 Z7 m8 Z8 m9 Z9 d
 d l: m Z y d  d l; m< Z< Wn e+ k
 re j=   Z< n Xe j> e?  Z@ ZA d ZB d   ZC eC   ZD d   ZE d   ZF d   ZG e/ jH d    ZI d d d  ZJ d d d  ZK d   ZL e d    ZM e1 jN d     ZO e1 jN d d!   ZP e1 jN d d"   ZQ d#   ZR d$ eS f d%     YZT e d&    ZU e d'    ZV d(   ZW d)   ZX e jY d*  ZZ d+   Z[ y d  d, l m\ Z\ Wn e+ k
 re] Z\ n Xe\ d-  e[  Z[ e6 r!d.   Z^ n	 d/   Z^ e e] d0   Z_ d1 d2  Z` e/ jH d3    Za e$ jb jc   d4   Zd d5 d6  Ze e jf d k rd8 d Un	 d9   Zg d:   Zh d  d;  Zi d<   Zj d=   Zk d>   Zl d?   Zm d d@  Zn dA   Zo d dB  Zp e dC dD   Zq dE   Zr dF   Zs dG   Zt eu e dH  rGe jv dI  Zw n. e] g Zx ex dJ  Zy ex dK  Zw e jz ey  dL ew _{ dC dM  Z| dN   Z} dO eS f dP     YZ~ dQ   Z i d dS 6d dU 6d dW 6d dY 6d d[ 6d d] 6d d_ 6d da 6d dc 6d de 6d
 df 6d
 dC 6Z dg   e j   D Z e j dh   e j   D  e j di   e j   D  dj   Z i d
 dk 6dl dm 6dn do 6dp dq 6dr ds 6dt du 6d dw 6Z i d
 dx 6dr dy 6d dz 6d d{ 6dl d| 6dn d} 6dp d~ 6Z e j d   e j   D  e j e  e j d   e j   D  d d  Z d   Z e7 rMe e e f d  Z n e e f d  Z d d  Z e d    Z e j d  Z d   Z d e j f d     YZ d   Z d e j k rte0 jf d  d k rte] Z d e j k r7d  d l Z d  d l m Z e j j j   o1e e j j j   e  Z n  e std  d l Z d  d l Z0 e j e0 j j j    qtn  d   Z e\ re\ d  e  Z n  d d d d d d d d d d d d d d d d d d g Z e/ jH e d   Z d   Z e d    Z d   Z d S(   i    (   t   print_functiont   divisiont   absolute_importN(   t   deque(   t   contextmanager(   t	   timedelta(   t   md5(   t   Number(   t   sleep(   t   import_modulei   (   t   cache_from_sourcet
   getargspect   invalidate_cachest   reload(   t   istask(   t   gen(   t   IOLoop(   t
   PollIOLoop(   t   Queuet   PY3t   PY2t   get_thread_identityt   unicode(   t   time(   t   thread_statet   __no_default__c          C` s   t  r t j j d  r d t j k r t j j d  }  t j	 |   } d g } d t j
 k rq | j d  n  | j |  n t } | S(   Nt   wint   PyPys)   distributed.worker.multiprocessing-methodt   distributedt   pkg_resources(   R   t   syst   platformt
   startswitht   versiont   daskt   configt   gett   multiprocessingt   get_contextt   modulest   appendt   set_forkserver_preload(   t   methodt   ctxt   preload(    (    s0   lib/python2.7/site-packages/distributed/utils.pyt   _initialize_mp_context@   s    (	c         C` sI   x t  |  d  r |  j }  q Wy |  j SWn t k
 rD t |   SXd S(   s   Get the name of a function.t   funcN(   t   hasattrR.   t   __name__t   AttributeErrort   str(   R.   (    (    s0   lib/python2.7/site-packages/distributed/utils.pyt   funcnameR   s    c         C` se   x^ t  r` y | t |   j k r% t  SWn t k
 r: Pn Xy |  j }  Wq t k
 r\ Pq Xq Wt S(   sE   
    Whether the function takes an argument with the given name.
    (   t   TrueR   t   argst	   TypeErrort   __wrapped__R1   t   False(   R.   t   argname(    (    s0   lib/python2.7/site-packages/distributed/utils.pyt   has_arg\   s    		c           C` s(   t  d k	 r  t  j t  j  d Sd Sd S(   s;   
    Get the maximum number of open files per process.
    i    i   N(   t   resourcet   Nonet	   getrlimitt   RLIMIT_NOFILE(    (    (    s0   lib/python2.7/site-packages/distributed/utils.pyt   get_fileno_limitn   s    c         C` s   t  j  | t  j  } zc y+ | j |  | f  | j   d } | SWn1 t k
 rv } t j d |  | | f t  | SXWd  | j   Xd  S(   Ni    sK   Couldn't detect a suitable IP address for reaching %r, defaulting to %r: %s(	   t   sockett
   SOCK_DGRAMt   connectt   getsocknamet   EnvironmentErrort   warningst   warnt   RuntimeWarningt   close(   t   hostt   portt   familyt   defaultt   sockt   ipt   e(    (    s0   lib/python2.7/site-packages/distributed/utils.pyt   _get_ipz   s     	s   8.8.8.8iP   c         C` s   t  |  | d t j d d S(   s   
    Get the local IP address through which the *host* is reachable.

    *host* defaults to a well-known Internet host (one of Google's public
    DNS servers).
    RK   RL   s	   127.0.0.1(   RP   R@   t   AF_INET(   RI   RJ   (    (    s0   lib/python2.7/site-packages/distributed/utils.pyt   get_ip   s    s   2001:4860:4860::8888c         C` s   t  |  | d t j d d S(   s-   
    The same as get_ip(), but for IPv6.
    RK   RL   s   ::1(   RP   R@   t   AF_INET6(   RI   RJ   (    (    s0   lib/python2.7/site-packages/distributed/utils.pyt   get_ipv6   s    c         C` sW   d d l  } x1 | j   |  D] } | j t j k r | j Sq Wt d |  f   d S(   s   
    Get the local IPv4 address of a network interface.

    KeyError is raised if the interface doesn't exist.
    ValueError is raised if the interface does no have an IPv4 address
    associated with it.
    i    Ns)   interface %r doesn't have an IPv4 address(   t   psutilt   net_if_addrsRK   R@   RQ   t   addresst
   ValueError(   t   ifnameRU   t   info(    (    s0   lib/python2.7/site-packages/distributed/utils.pyt   get_ip_interface   s
    c          g` s#   y	 d  VWn |  k
 r } n Xd  S(   N(    (   t
   exceptionsRO   (    (    s0   lib/python2.7/site-packages/distributed/utils.pyt   ignoring   s    	c         g` sh   t  j |    } g  } x= | j   sT t |    | j   V} | j |  Wd QXq Wt  j |   d S(   s    Process list of coroutines, ignoring certain exceptions

    >>> coroutines = [cor(...) for ...]  # doctest: +SKIP
    >>> x = yield ignore_exceptions(coroutines, TypeError)  # doctest: +SKIP
    N(   R   t   WaitIteratort   doneR]   t   nextR(   t   Return(   t
   coroutinesR\   t   wait_iteratort   resultst   result(    (    s0   lib/python2.7/site-packages/distributed/utils.pyt   ignore_exceptions   s    c         #` s   t  j |     g  |  D] } d ^ q } xg  j   s y  j   V} Wn6 t k
 r t  j    f d    } |     n X| |  j <q+ Wt  j |   d S(   s"   Wait on many tasks at the same time

    Err once any of the tasks err.

    See https://github.com/tornadoweb/tornado/issues/1546

    Parameters
    ----------
    args: futures to wait for
    quiet_exceptions: tuple, Exception
        Exception types to avoid logging if they fail
    c          3` s;   x4 t   j  D]# }  y	 |  VWq   k
 r2 q Xq Wd S(   s    Watch unfinished tasks

                Otherwise if they err they get logged in a way that is hard to
                control.  They need some other task to watch them so that they
                are not orphaned
                N(   t   listt   _unfinished(   t   task(   t   quiet_exceptionst   tasks(    s0   lib/python2.7/site-packages/distributed/utils.pyt   quiet   s
    	N(	   R   R^   R<   R_   R`   t	   Exceptiont	   coroutinet   current_indexRa   (   R5   Rj   t   _Rd   Re   Rl   (    (   Rj   Rk   s0   lib/python2.7/site-packages/distributed/utils.pyt   All   s    c         #` s   t  j |     g  |  D] } d ^ q } xh  j   s y  j   V} Wn6 t k
 r t  j    f d    } |     n X| |  j <Pq+ Wt  j |   d S(   s   Wait on many tasks at the same time and return when any is finished

    Err once any of the tasks err.

    Parameters
    ----------
    args: futures to wait for
    quiet_exceptions: tuple, Exception
        Exception types to avoid logging if they fail
    c          3` s;   x4 t   j  D]# }  y	 |  VWq   k
 r2 q Xq Wd S(   s    Watch unfinished tasks

                Otherwise if they err they get logged in a way that is hard to
                control.  They need some other task to watch them so that they
                are not orphaned
                N(   Rg   Rh   (   Ri   (   Rj   Rk   (    s0   lib/python2.7/site-packages/distributed/utils.pyRl     s
    	N(	   R   R^   R<   R_   R`   Rm   Rn   Ro   Ra   (   R5   Rj   Rp   Rd   Re   Rl   (    (   Rj   Rk   s0   lib/python2.7/site-packages/distributed/utils.pyt   Any   s    c      	   ` s{  t  rQ t |  t   r' t |  d t  sB t |  d  rQ |  j j rQ t d   n  y" |  j j   rr t d   n  Wn t	 k
 r n X j
 d d	   t j    t    d	 g  t g  t j          f d    } |  j |   d	 k	 r1 j   sQt j d  f   qQn  x  j   sP j d  q4W d rot j  d   n  d Sd	 S(
   s;   
    Run coroutine in loop running in separate thread.
    t   _closingt   asyncio_loops   IOLoop is closedt   callback_timeoutc          3` s   z yw  t    k r$ t d   n  t j Vt t _       }   d  k	 rn t j t	 d   |   }  n  |  V d <Wn# t
 k
 r } t j    d <n XWd  t t _  j   Xd  S(   Ns)   sync() called from thread of running loopt   secondsi    (   R   t   RuntimeErrorR   t   momentR4   R   t   asynchronousR<   t   with_timeoutR   Rm   R   t   exc_infoR8   t   set(   t   futuret   exc(   R5   RO   t   errorR.   t   kwargst   main_tidRe   t   timeout(    s0   lib/python2.7/site-packages/distributed/utils.pyt   f2  s     		s   timed out after %s s.i
   i    N(   R   t
   isinstancet   getattrR8   R/   Rt   t   _closedRw   t	   is_closedR1   t   popR<   t	   threadingt   EventR   R   Rn   t   add_callbackt   waitt   TimeoutErrort   is_sett   sixt   reraise(   t   loopR.   R5   R   R   (    (   R5   RO   R   R.   R   R   Re   R   s0   lib/python2.7/site-packages/distributed/utils.pyt   sync  s0    !			-
t
   LoopRunnerc           B` s   e  Z d  Z e j   Z e j   Z d e
 d  Z d   Z d   Z d d  Z d   Z d   Z d   Z d	   Z e d
    Z RS(   sg  
    A helper to start and stop an IO loop in a controlled way.
    Several loop runners can associate safely to the same IO loop.

    Parameters
    ----------
    loop: IOLoop (optional)
        If given, this loop will be re-used, otherwise an appropriate one
        will be looked up or created.
    asynchronous: boolean (optional, default False)
        If false (the default), the loop is meant to run in a separate
        thread and will be started if necessary.
        If true, the loop is meant to run in the thread this
        object is instantiated from, and will not be started automatically.
    c         C` s   t  j   } | d  k rB | r* | |  _ n t    |  _ t |  _ n | |  _ t |  _ | |  _ d  |  _ t |  _	 |  j
  |  j j |  j d  Wd  QXd  S(   Ni    (   i    N(   R   t   currentR<   t   _loopR4   t   _should_close_loopR8   t   _asynchronoust   _loop_threadt   _startedt   _lockt
   _all_loopst
   setdefault(   t   selfR   Ry   R   (    (    s0   lib/python2.7/site-packages/distributed/utils.pyt   __init__e  s    					
c         C` s   |  j   |  j   Wd QXd S(   s   
        Start the IO loop if required.  The loop is run in a dedicated
        thread.

        If the loop is already running, this method does nothing.
        N(   R   t   _start_unlocked(   R   (    (    s0   lib/python2.7/site-packages/distributed/utils.pyt   startx  s    
c         ` s  |  j  s t  |  j |  j \ } } |  j sG | d  k	 sG | d k rn | d | f |  j |  j <t |  _  d  S|  j d  k s t  | d k s t  t j	    t j	     d  g  d  g    f d    |  j     f d  } t j
 d | d d  } t | _ | j    j d d	  t |  _   d } | | k	 r  j d
  t  d t  st  d t  st d  d   n   d  n  | d d  f |  j |  j <n?  d d  k st    | |  _ | d |  f |  j |  j <d  S(   Ni    i   c           ` s   t  j     d < j   d  S(   Ni    (   R   t   current_threadR|   (    (   t	   in_threadt   loop_evt(    s0   lib/python2.7/site-packages/distributed/utils.pyt   loop_cb  s    c         ` sQ   |  j    z2 y |  j   Wn t k
 r= } |  d <n XWd    j   Xd  S(   Ni    (   R   R   Rm   R|   (   R   RO   (   t   done_evtR   t	   start_exc(    s0   lib/python2.7/site-packages/distributed/utils.pyt   run_loop  s     t   targett   names   IO loopR   i
   i   s   not an exception(   R   t   AssertionErrorR   R   R   R<   R4   R   R   R   t   Threadt   daemonR   R   R   Rw   Rm   R6   (   R   t   countt   real_runnerR   t   threadt   actual_thread(    (   R   R   R   R   R   s0   lib/python2.7/site-packages/distributed/utils.pyR     s>    !					
	
	i
   c         C` s!   |  j   |  j |  Wd QXd S(   sv   
        Stop and close the loop if it was created by us.
        Otherwise, just mark this object "stopped".
        N(   R   t   _stop_unlocked(   R   R   (    (    s0   lib/python2.7/site-packages/distributed/utils.pyt   stop  s    
c         C` s   |  j  s d  St |  _  |  j |  j \ } } | d k rU | d | f |  j |  j <n; | d k sg t  |  j |  j =| d  k	 r | j |  n  d  S(   Ni   (   R   R8   R   R   R   R<   t
   _real_stop(   R   R   R   R   (    (    s0   lib/python2.7/site-packages/distributed/utils.pyR     s    		c         C` s   |  j  d  k	 s t  |  j  d  k	 r zM |  j j |  j j  |  j  j d |  t t   |  j j	   Wd  QXWd  d  |  _  Xn  d  S(   NR   (
   R   R<   R   R   R   R   t   joinR]   t   KeyErrorRH   (   R   R   (    (    s0   lib/python2.7/site-packages/distributed/utils.pyR     s    c         C` s   |  j  S(   sP   
        Return True between start() and stop() calls, False otherwise.
        (   R   (   R   (    (    s0   lib/python2.7/site-packages/distributed/utils.pyt
   is_started  s    c         O` sU   |  j  r t |  j | | |  S|  j   z t |  j | | |  SWd |  j   Xd S(   s   
        Convenience helper: start the loop if needed,
        run sync(func, *args, **kwargs), then stop the loop again.
        N(   R   R   R   R   R   (   R   R.   R5   R   (    (    s0   lib/python2.7/site-packages/distributed/utils.pyt   run_sync  s    	
c         C` s   |  j  S(   N(   R   (   R   (    (    s0   lib/python2.7/site-packages/distributed/utils.pyR     s    N(   R0   t
   __module__t   __doc__t   weakreft   WeakKeyDictionaryR   R   t   LockR   R<   R8   R   R   R   R   R   R   R   R   t   propertyR   (    (    (    s0   lib/python2.7/site-packages/distributed/utils.pyR   P  s   	
	5				c       	   k` s   i  } x9 |  D]1 } y t  t |  | | <Wq t k
 r= q Xq Wx* |  j   D] \ } } t t | |  qO Wz	 d  VWd  xM |  D]E } y | | } Wn t k
 r t t |  q Xt t | |  q WXd  S(   N(   R   R   R1   t   itemst   setattrR   t   delattr(   R   t   oldt   kt   v(    (    s0   lib/python2.7/site-packages/distributed/utils.pyt   set_thread_state  s     	c         c` ss   t  j j t j   |   } t | d   } | j |  Wd  QXz	 | VWd  t  j j |  rn t  j |  n  Xd  S(   Nt   w(	   t   ost   pathR   t   tempfilet
   gettempdirt   opent   writet   existst   remove(   t   filenamet   textt   fnR   (    (    s0   lib/python2.7/site-packages/distributed/utils.pyt   tmp_text  s    	c         C` s!   x |  j    s |  j   q Wd  S(   N(   t   emptyt
   get_nowait(   t   q(    (    s0   lib/python2.7/site-packages/distributed/utils.pyt   clear_queue  s    c          C` s<   d t  j k r t Sd d l m }  t |    d d  d k	 S(   sX    Determine if we're running within an IPython kernel

    >>> is_kernel()
    False
    t   IPythoni    (   t   get_ipythont   kernelN(   R   R'   R8   R   R   R   R<   (   R   (    (    s0   lib/python2.7/site-packages/distributed/utils.pyt	   is_kernel  s    s   [a-f]+c         C` sm  t  |   t k r! |  j   }  n  t  |   t k r@ |  d }  n  y|  j d  } | d d j   s | d j d  d j d  } n
 | d } xX | d D]L } | j   r t |  d k o t j	 |  d k	 r | d | 7} q Pq Wt |  d k rt j	 d |  rd	 S| d d
 k rO| j d  j   d j d  d } n  | SWn t k
 rhd SXd S(   s5  
    >>> key_split('x')
    'x'
    >>> key_split('x-1')
    'x'
    >>> key_split('x-1-2-3')
    'x'
    >>> key_split(('x-2', 1))
    'x'
    >>> key_split("('x-2', 1)")
    'x'
    >>> key_split("('x', 1)")
    'x'
    >>> key_split('hello-world-1')
    'hello-world'
    >>> key_split(b'hello-world-1')
    'hello-world'
    >>> key_split('ae05086432ca935f6eba409a8ecd4896')
    'data'
    >>> key_split('<module.submodule.myclass object at 0xdaf372')
    'myclass'
    >>> key_split(None)
    'Other'
    >>> key_split('x-abcdefab')  # ignores hex
    'x'
    i    t   -t   ,s   '("i   i   i    s   [a-f0-9]{32}t   datat   <s   <>t   .it   OtherN(   t   typet   bytest   decodet   tuplet   splitt   isalphat   stript   lent   hex_patternt   matchR<   t   reRm   (   t   st   wordsRe   t   word(    (    s0   lib/python2.7/site-packages/distributed/utils.pyt	   key_split,  s*    #
($)(   t	   lru_cachei c         C` s   t  |   } | t k r  |  d S| t k r |  d d k rY |  j d d  d j d  St |   d k r t j d |   r d S|  d d	 k r |  j d
  j   d j d  d S|  Sn  | t k r t	 |  j
    Sd Sd S(   s  A more fine-grained version of key_split

        >>> key_split_group('x')
        'x'
        >>> key_split_group('x-1')
        'x-1'
        >>> key_split_group('x-1-2-3')
        'x-1-2-3'
        >>> key_split_group(('x-2', 1))
        'x-2'
        >>> key_split_group("('x-2', 1)")
        'x-2'
        >>> key_split_group('hello-world-1')
        'hello-world-1'
        >>> key_split_group(b'hello-world-1')
        'hello-world-1'
        >>> key_split_group('ae05086432ca935f6eba409a8ecd4896')
        'data'
        >>> key_split_group('<module.submodule.myclass object at 0xdaf372')
        'myclass'
        >>> key_split_group(None)
        'Other'
        >>> key_split_group('x-abcdefab')  # ignores hex
        'x-abcdefab'
        i    t   (R   i   s   ()"'i    s   [a-f0-9]{32}R   R   s   <>R   iR   N(   R   R   R2   R   R   R   R   R   R   t   key_split_groupR   (   t   xt   typ(    (    s0   lib/python2.7/site-packages/distributed/utils.pyR   l  s    $$c         C` s   t  |   } | t k r  |  d S| t k s8 | t k r |  d d k re |  j d d  d j d  St |   d k r t j d |   r d S|  d d	 k r |  j d
  j   d j d  d S|  Sn d Sd S(   s  A more fine-grained version of key_split

        >>> key_split_group('x')
        'x'
        >>> key_split_group('x-1')
        'x-1'
        >>> key_split_group('x-1-2-3')
        'x-1-2-3'
        >>> key_split_group(('x-2', 1))
        'x-2'
        >>> key_split_group("('x-2', 1)")
        'x-2'
        >>> key_split_group('hello-world-1')
        'hello-world-1'
        >>> key_split_group(b'hello-world-1')
        'hello-world-1'
        >>> key_split_group('ae05086432ca935f6eba409a8ecd4896')
        'data'
        >>> key_split_group('<module.submodule.myclass object at 0xdaf372')
        'myclass'
        >>> key_split_group(None)
        'Other'
        >>> key_split_group('x-abcdefab')  # ignores hex
        'x-abcdefab'
        i    R   R   i   s   ()"'i    s   [a-f0-9]{32}R   R   s   <>R   iR   N(	   R   R   R2   R   R   R   R   R   R   (   R   R   (    (    s0   lib/python2.7/site-packages/distributed/utils.pyR     s    $$c         c` s   d d l  m } y	 d  VWnv | t j f k
 r8   nZ t k
 r } y t j |  Wn t k
 rk n X|  r d d  l }  |  j	   n    n Xd  S(   Ni   (   t   CommClosedErrori    (
   t   commR   R   Ra   Rm   t   loggert	   exceptionR6   t   pdbt	   set_trace(   R   R   RO   (    (    s0   lib/python2.7/site-packages/distributed/utils.pyt
   log_errors  s    	R   c         C` s   t  |  t  r' t t |  j    }  n  d } t j |  } x< | j D]1 } t  | t j  rF | j	 } | j
 |   qF qF W| S(   sK   
    Change all StreamHandlers for the given logger to the given level
    N(   R   R2   R   t   loggingt   upperR<   t	   getLoggert   handlerst   StreamHandlert   levelt   setLevel(   R   t   rootR   R   t   handler(    (    s0   lib/python2.7/site-packages/distributed/utils.pyt   silence_logging  s    	c         C` s{   t  j t  j g } x\ | D]T } y t  j |  d | t  j  } Wn t  j k
 r\ } | } q X| d d d Sq W|  d S(   s    Ensure that address is an IP address

    Examples
    --------
    >>> ensure_ip('localhost')
    '127.0.0.1'
    >>> ensure_ip('123.123.123.123')  # pass through IP addresses
    '123.123.123.123'
    i  i    i   N(   R@   RQ   RS   t   getaddrinfot   SOCK_STREAMt   gaierror(   t   hostnamet   familiest   famRd   RO   R~   (    (    s0   lib/python2.7/site-packages/distributed/utils.pyt	   ensure_ip  s    
c          ` s   t  j   \ }  }   t j j d d  t j j d d  t j j d d  t j j d d  g } x/   r t   f d   | D  r   j   qf W  S(	   NR   t   workert	   schedulert   tornados   gen.pyt
   concurrentt   futuresc         3` s$   |  ] } |   j  j j k Vq d  S(   N(   t   tb_framet   f_codet   co_filename(   t   .0t   b(   t   exc_traceback(    s0   lib/python2.7/site-packages/distributed/utils.pys	   <genexpr>  s    (   R   R{   R   R   R   t   anyt   tb_next(   t   exc_typet	   exc_valuet   bad(    (   R  s0   lib/python2.7/site-packages/distributed/utils.pyt   get_traceback  s    i'  c         C` su   t  t |    | k rm y! t |   d t |   |   SWqq t k
 ri t d t |   t |   |   SXn |  Sd S(   s1    Truncate exception to be about a certain length s   Long error messageN(   R   R2   R   Rm   (   RO   t   n(    (    s0   lib/python2.7/site-packages/distributed/utils.pyt   truncate_exception  s    !$i   s   def queue_to_iterator(q):
        while True:
            result = q.get()
            if isinstance(result, StopIteration):
                return result.value
            yield result
        c         c` s:   x3 t  r5 |  j   } t | t  r- |  n  | Vq Wd  S(   N(   R4   R$   R   t   StopIteration(   R   Re   (    (    s0   lib/python2.7/site-packages/distributed/utils.pyt   queue_to_iterator-  s
    		c         C` s"   x |  D] } | j  |  q Wd  S(   N(   t   put(   t   seqR   t   item(    (    s0   lib/python2.7/site-packages/distributed/utils.pyt   _dump_to_queue5  s    c         C` sD   t  d |  } t j d t d |  | f  } t | _ | j   | S(   Nt   maxsizeR   R5   (   R   R   R   R$  R4   R   R   (   R"  R%  R   t   t(    (    s0   lib/python2.7/site-packages/distributed/utils.pyt   iterator_to_queue:  s
    	
c         C` s6   t  |   } | t k s$ | t k r( |  St |   Sd S(   s    Convert an object to a string.

    Examples
    --------

    >>> tokey(b'x')
    'x'
    >>> tokey('x')
    'x'
    >>> tokey(1)
    '1'
    N(   R   R   R   R2   (   t   oR   (    (    s0   lib/python2.7/site-packages/distributed/utils.pyt   tokeyD  s    c         C` sA   t  |   } | t k	 r= | t k	 r= t d | |  f   n  d S(   s,   Validate a key as received on a stream.
    s"   Unexpected key type %s (value: %r)N(   R   R   R   R6   (   R   R   (    (    s0   lib/python2.7/site-packages/distributed/utils.pyt   validate_keyX  s    c         C` s^   t  |   p] t |   t k r3 t t t |    p] t |   t k o] t t t |  j     S(   s!    Possibly contains a nested task (   R   R   Rg   R  t   mapt   _maybe_complext   dictt   values(   Ri   (    (    s0   lib/python2.7/site-packages/distributed/utils.pyR,  `  s
    c         ` s   t  |   t k r5 g  |  D] } t |     ^ q St  |   t k rd    f d   |  j   D St |   r |  d f t    f d   |  d D  Sy& |    k s |   k r t |   SWn t k
 r n X|  S(   Nc         ` s+   i  |  ]! \ } } t  |     |  q S(    (   t   convert(   R  R   R   (   t   dskt   extra_values(    s0   lib/python2.7/site-packages/distributed/utils.pys
   <dictcomp>o  s   	 i    c         3` s!   |  ] } t  |     Vq d  S(   N(   R/  (   R  R   (   R0  R1  (    s0   lib/python2.7/site-packages/distributed/utils.pys	   <genexpr>q  s    i   (	   R   Rg   R/  R-  R   R   R   R)  R6   (   Ri   R0  R1  R   (    (   R0  R1  s0   lib/python2.7/site-packages/distributed/utils.pyR/  k  s    #,c         ` s      f d     j    D S(   Nc         ` s1   i  |  ]' \ } } t  |     t |   q S(    (   R/  R)  (   R  R   R   (   R0  R1  (    s0   lib/python2.7/site-packages/distributed/utils.pys
   <dictcomp>{  s   	 (   R   (   R0  R1  (    (   R0  R1  s0   lib/python2.7/site-packages/distributed/utils.pyt	   str_graphz  s    c         C` s   |  j    d k r d Sd } x t r |  j |  } | s> d S| | } yB | j |  } |  j |  j    t |  | t |   d SWn t k
 r n X| t |  } q Wd S(   s   Seek current file to next byte after a delimiter bytestring

    This seeks the file to the next byte following the delimiter.  It does
    not return anything.  Use ``file.tell()`` to see location afterwards.

    Parameters
    ----------
    file: a file
    delimiter: bytes
        a delimiter like ``b'
'`` or message sentinel
    blocksize: int
        Number of bytes to read from the file at once.
    i    Nt    (   t   tellR4   t   readt   indext   seekR   RX   (   t   filet	   delimitert	   blocksizet   lastR   t   fullt   i(    (    s0   lib/python2.7/site-packages/distributed/utils.pyt   seek_delimiter~  s    	
+c         C` s   | r} |  j  |  t |  | d  |  j   } | | | 8} |  j  | |  t |  | d  |  j   } | } | | } n  |  j  |  |  j |  } | S(   s%   Read a block of bytes from a file

    Parameters
    ----------
    f: file
        File-like object supporting seek, read, tell, etc..
    offset: int
        Byte offset to start read
    length: int
        Number of bytes to read
    delimiter: bytes (optional)
        Ensure reading starts and stops at delimiter bytestring

    If using the ``delimiter=`` keyword argument we ensure that the read
    starts and stops at delimiter boundaries that follow the locations
    ``offset`` and ``offset + length``.  If ``offset`` is zero then we
    start at zero.  The bytestring returned WILL include the
    terminating delimiter string.

    Examples
    --------

    >>> from io import BytesIO  # doctest: +SKIP
    >>> f = BytesIO(b'Alice, 100\nBob, 200\nCharlie, 300')  # doctest: +SKIP
    >>> read_block(f, 0, 13)  # doctest: +SKIP
    b'Alice, 100\nBo'

    >>> read_block(f, 0, 13, delimiter=b'\n')  # doctest: +SKIP
    b'Alice, 100\nBob, 200\n'

    >>> read_block(f, 10, 10, delimiter=b'\n')  # doctest: +SKIP
    b'Bob, 200\nCharlie, 300'
    i   i   i   i   (   R7  R>  R4  R5  (   R   t   offsett   lengthR9  R   t   endR   (    (    s0   lib/python2.7/site-packages/distributed/utils.pyt
   read_block  s    "R3  c         c` s   d |  j  d  }  t j |   \ } } t j |  t j |  | Vt j j |  r t j j |  r{ t	 j
 |  q y t j |  Wq t k
 r q Xn  d  S(   NR   (   t   lstripR   t   mkstempR   RH   R   R   R   t   isdirt   shutilt   rmtreet   OSError(   t	   extensiont   handleR   (    (    s0   lib/python2.7/site-packages/distributed/utils.pyt   tmpfile  s    c         C` s   t  |  t  r |  St  |  t  r, |  j   St  |  t  sP t rZ t  |  t  rZ t |   St |  d  rs |  j   St	 d |    d S(   ss    Turn string or bytes to bytes

    >>> ensure_bytes('123')
    b'123'
    >>> ensure_bytes(b'123')
    b'123'
    t   encodes<   Object %s is neither a bytes object nor has an encode methodN(
   R   R   t
   memoryviewt   tobytest	   bytearrayR   t   bufferR/   RL  R6   (   R   (    (    s0   lib/python2.7/site-packages/distributed/utils.pyt   ensure_bytes  s    
$

c         C` sg   t  |  } d } g  } xH | D]@ } |  | | | } t | d  \ } } | j t |   q W| S(   s  

    >>> divide_n_among_bins(12, [1, 1])
    [6, 6]
    >>> divide_n_among_bins(12, [1, 2])
    [4, 8]
    >>> divide_n_among_bins(12, [1, 2, 1])
    [3, 6, 3]
    >>> divide_n_among_bins(11, [1, 2, 1])
    [2, 6, 3]
    >>> divide_n_among_bins(11, [.1, .2, .1])
    [2, 6, 3]
    g        i   (   t   sumt   divmodR(   t   int(   R  t   binst   totalt   acct   outR  t   now(    (    s0   lib/python2.7/site-packages/distributed/utils.pyt   divide_n_among_bins  s    c         C` s    t  |   }  t |   t |   S(   N(   Rg   RR  R   (   R"  (    (    s0   lib/python2.7/site-packages/distributed/utils.pyt   mean  s    t   is_finalizingc         C` s   |    S(   N(    (   R\  (    (    s0   lib/python2.7/site-packages/distributed/utils.pyt   shutting_down  s    c         C` s   t  |  d <d  S(   Ni    (   R4   (   t   l(    (    s0   lib/python2.7/site-packages/distributed/utils.pyt   _at_shutdown  s    c         C` s   |  d S(   Ni    (    (   R^  (    (    s0   lib/python2.7/site-packages/distributed/utils.pyR]  "  s    s  
    Whether the interpreter is currently shutting down.
    For use in finalizers, __del__ methods, and similar; it is advised
    to early bind this function rather than look it up when calling it,
    since at shutdown module globals may be cleared.
    c         C` sV   t  j  t  j t  j  } | j |  d f  | j d  | j   d } | j   | S(   s    Return a probably-open port

    There is a chance that this port will be taken by the operating system soon
    after returning from this function.
    i    i   (   R@   RQ   R  t   bindt   listenRC   RH   (   RI   R   RJ   (    (    s0   lib/python2.7/site-packages/distributed/utils.pyt	   open_port0  s    
c         C` s  t  j j |   \ } } t  j j |  \ } } g  } d } | d	 k rp | t j k r` | } n  | j |  n  | d k r t |   } t t	   t  j
 |  Wd QXn  | d
 k rF|  t j k r t j j d |   n  | d k r'd d l } | j |   }	 x= |	 D] }
 | j |
 j  q
WqF| d k rF| j |  qFn  g  } | set j d |  n t   | d k	 rt j j d |  n  zA x: | D]2 } t j d | |  | j t t |    qWWd | d k	 rt j j
 |  n  X| S(   s,    Loads modules for a file (.py, .zip, .egg) s   .pyNs   .eggs   .zips   .pyzi    s   Found nothing to import from %ss   Reload module %s from %s file(   s   .py(   s   .eggs   .zips   .pyz(   s   .zips   .pyz(   R   R   R   t   splitextR<   R   R(   R
   R]   RH  R   t   insertR   t   find_distributionst   project_nameR   t   warningR   RZ   R   R	   (   R   t	   directoryR   R   t   extt   names_to_importt   tmp_python_patht
   cache_fileR   t   pkgst   pkgt   loaded(    (    s0   lib/python2.7/site-packages/distributed/utils.pyt   import_file?  sF    	!t
   itemgetterc           B` s/   e  Z d  Z d Z d   Z d   Z d   Z RS(   s   A picklable itemgetter.

    Examples
    --------
    >>> data = [0, 1, 2]
    >>> get_1 = itemgetter(1)
    >>> get_1(data)
    1
    R6  c         C` s   | |  _  d  S(   N(   R6  (   R   R6  (    (    s0   lib/python2.7/site-packages/distributed/utils.pyR   x  s    c         C` s   | |  j  S(   N(   R6  (   R   R   (    (    s0   lib/python2.7/site-packages/distributed/utils.pyt   __call__{  s    c         C` s   t  |  j f f S(   N(   Rq  R6  (   R   (    (    s0   lib/python2.7/site-packages/distributed/utils.pyt
   __reduce__~  s    (   R6  (   R0   R   R   t	   __slots__R   Rr  Rs  (    (    (    s0   lib/python2.7/site-packages/distributed/utils.pyRq  k  s
   			c         C` s   |  d k r d |  d S|  d k r0 d |  d S|  d k rH d |  d S|  d k r` d |  d S|  d	 k rx d
 |  d Sd |  S(   s*   Format bytes as text

    >>> format_bytes(1)
    '1 B'
    >>> format_bytes(1234)
    '1.23 kB'
    >>> format_bytes(12345678)
    '12.35 MB'
    >>> format_bytes(1234567890)
    '1.23 GB'
    >>> format_bytes(1234567890000)
    '1.23 TB'
    >>> format_bytes(1234567890000000)
    '1.23 PB'
    g  4&kCs   %0.2f PBg   mBs   %0.2f TBg    eAs   %0.2f GBg    .As   %0.2f MBg     @@s   %0.2f kBi  s   %d B(    (   R  (    (    s0   lib/python2.7/site-packages/distributed/utils.pyt   format_bytes  s    i
   t   kBi   t   MBi	   t   GBi   t   TBi   t   PBi   t   KiBi   t   MiBi   t   GiBi(   t   TiBi2   t   PiBt   Bc         C` s%   i  |  ] \ } } | | j     q S(    (   t   lower(   R  R   R   (    (    s0   lib/python2.7/site-packages/distributed/utils.pys
   <dictcomp>  s   	 c         C` s5   i  |  ]+ \ } } | r d  | k r | | d  q S(   R=  i    (    (   R  R   R   (    (    s0   lib/python2.7/site-packages/distributed/utils.pys
   <dictcomp>  s   	 c         C` s5   i  |  ]+ \ } } | r d  | k r | | d   q S(   R=  i(    (   R  R   R   (    (    s0   lib/python2.7/site-packages/distributed/utils.pys
   <dictcomp>  s   	 c         C` s   |  j  d d  }  |  d j   s/ d |  }  n  x8 t t |   d d d  D] } |  | j   sL PqL qL W| d } |  |  } |  | } t |  } t | j   } | | } t |  S(   s   Parse byte string to numbers

    >>> parse_bytes('100')
    100
    >>> parse_bytes('100 MB')
    100000000
    >>> parse_bytes('100M')
    100000000
    >>> parse_bytes('5kB')
    5000
    >>> parse_bytes('5.4 kB')
    5400
    >>> parse_bytes('1kiB')
    1024
    >>> parse_bytes('1e6')
    1000000
    >>> parse_bytes('1e6 kB')
    1000000000
    >>> parse_bytes('MB')
    1000000
    t    R3  i    t   1i   i(	   t   replacet   isdigitt   rangeR   R   t   floatt
   byte_sizesR  RT  (   R   R=  R6  t   prefixt   suffixR  t
   multiplierRe   (    (    s0   lib/python2.7/site-packages/distributed/utils.pyt   parse_bytes  s    #



R   gMbP?t   msgư>t   usg&.>t   nsi<   t   mi  t   hi   t   dt   secondt   minutet   hourt   dayt   millisecondt   microsecondt
   nanosecondc         C` s#   i  |  ] \ } } | | d   q S(   R   (    (   R  R   R   (    (    s0   lib/python2.7/site-packages/distributed/utils.pys
   <dictcomp>  s   	 c         C` s%   i  |  ] \ } } | | j     q S(    (   R   (   R  R   R   (    (    s0   lib/python2.7/site-packages/distributed/utils.pys
   <dictcomp>  s   	 Rv   c   	      C` s  t  |  t  r |  j   St  |  t  r7 t |   }  n  |  j d d  }  |  d j   sf d |  }  n  x8 t t |   d d d  D] } |  | j	   s Pq q W| d } |  |  } |  | p | } t
 |  } t | j   } | | } t |  | k rt |  } n  | S(   s   Parse timedelta string to number of seconds

    Examples
    --------
    >>> parse_timedelta('3s')
    3
    >>> parse_timedelta('3.5 seconds')
    3.5
    >>> parse_timedelta('300ms')
    0.3
    >>> parse_timedelta(timedelta(seconds=3))  # also supports timedeltas
    3
    R  R3  i    R  i   i(   R   R   t   total_secondsR   R2   R  R  R  R   R   R  t   timedelta_sizesR  RT  (	   R   RL   R=  R6  R  R  R  R  Re   (    (    s0   lib/python2.7/site-packages/distributed/utils.pyt   parse_timedelta  s&    
#


c         ` s   g  | D] } t  d   | D  ^ q } t  d   |  D  }  t  d   t t |   |   D  } d d t |   |     t  |   } d d j d   | D  } d	 j   f d
   | D  } d	 j | | | | | g  S(   s   Formats an ascii table for given columns and rows.

    Parameters
    ----------
    columns : list
        The column names
    rows : list of tuples
        The rows in the table. Each tuple must be the same length as
        ``columns``.
    c         s` s   |  ] } t  |  Vq d  S(   N(   R2   (   R  R=  (    (    s0   lib/python2.7/site-packages/distributed/utils.pys	   <genexpr>'  s    c         s` s   |  ] } t  |  Vq d  S(   N(   R2   (   R  R=  (    (    s0   lib/python2.7/site-packages/distributed/utils.pys	   <genexpr>(  s    c         s` s9   |  ]/ \ } } t  t  t t |   t |   Vq d  S(   N(   t   maxR+  R   (   R  R   t   c(    (    s0   lib/python2.7/site-packages/distributed/utils.pys	   <genexpr>)  s    t   |s	    %%-%ds |s   +%s+t   +c         s` s   |  ] } d  | d Vq d S(   R   i   N(    (   R  R   (    (    s0   lib/python2.7/site-packages/distributed/utils.pys	   <genexpr>,  s    s   
c         3` s   |  ] }   | Vq d  S(   N(    (   R  t   r(   t   row_template(    s0   lib/python2.7/site-packages/distributed/utils.pys	   <genexpr>-  s    (   R   t   zipR   R   (   t   columnst   rowsR  t   widthst   headert   barR   (    (   R  s0   lib/python2.7/site-packages/distributed/utils.pyt
   asciitable  s    )%c         C` sh   t  |  |  r t |   St  |  t  r] |  j d k r> |  j St j t j	 |  j |  j  Sn |  j
 Sd S(   s*    Number of bytes of a frame or memoryview N(   R   R   RM  t   shapeR<   t   itemsizet	   functoolst   reducet   operatort   mult   nbytes(   t   framet   _bytes_like(    (    s0   lib/python2.7/site-packages/distributed/utils.pyR  3  s    
c         C` sC   t  |  |  r t |   Sy |  j SWn t k
 r> t |   SXd S(   s*    Number of bytes of a frame or memoryview N(   R   R   R  R1   (   R  R  (    (    s0   lib/python2.7/site-packages/distributed/utils.pyR  B  s    
c         C` s<   t  j d k r" t  j j |  |  St  j j |  | |  Sd S(   s   
    Wrapper around tornado.IOLoop.PeriodicCallback, for compatibility
    with removal of the `io_loop` parameter in Tornado 5.0.
    i   N(   i   (   R  t   version_infot   ioloopt   PeriodicCallback(   t   callbackt   callback_timet   io_loop(    (    s0   lib/python2.7/site-packages/distributed/utils.pyR  M  s    c         c` sB   t    } d  Vt    } | | |  k r> t d | | |  n  d  S(   Ns   TIME WARNING(   R   t   print(   t   durationR   R   RA  (    (    s0   lib/python2.7/site-packages/distributed/utils.pyt	   time_warnX  s
    		c         C` s   x  t  j j |   s" t d  q Wxg t d  D]Y } y/ t |    } | |  } Wd QX| rd | SWn t t f k
 r~ n Xt d  q0 Wd S(   sB    Reads a JSON file from disk that may be being written as we read g{Gz?i
   Ng?(   R   R   R   R   R  R   RX   R   (   R   t   loadR=  R   t   cfg(    (    s0   lib/python2.7/site-packages/distributed/utils.pyt   json_load_robusta  s    c         C` s8   |  d k r d |  S|  d k r, d |  d Sd |  d S(   s    format integers as time

    >>> format_time(1)
    '1.00 s'
    >>> format_time(0.001234)
    '1.23 ms'
    >>> format_time(0.00012345)
    '123.45 us'
    >>> format_time(123.456)
    '123.46 s'
    i   s   %.2f sgMbP?s   %.2f msg     @@s   %.2f usg    .A(    (   R  (    (    s0   lib/python2.7/site-packages/distributed/utils.pyt   format_timep  s
    t   DequeHandlerc           B` sD   e  Z d  Z e j   Z d   Z d   Z d   Z e	 d    Z
 RS(   s5    A logging.Handler that records records into a deque c         O` sQ   | j  d d  } t d |  |  _ t t |   j | |   |  j j |   d  S(   NR  i'  t   maxlen(   R   R   t   superR  R   t
   _instancest   add(   R   R5   R   R  (    (    s0   lib/python2.7/site-packages/distributed/utils.pyR     s    c         C` s   |  j  j |  d  S(   N(   R   R(   (   R   t   record(    (    s0   lib/python2.7/site-packages/distributed/utils.pyt   emit  s    c         C` s   |  j  j   d S(   s)   
        Clear internal storage.
        N(   R   t   clear(   R   (    (    s0   lib/python2.7/site-packages/distributed/utils.pyR    s    c         C` s(   x! t  |  j  D] } | j   q Wd S(   sG   
        Clear the internal storage of all live DequeHandlers.
        N(   Rg   R  R  (   t   clst   inst(    (    s0   lib/python2.7/site-packages/distributed/utils.pyt   clear_all_instances  s    (   R0   R   R   R   t   WeakSetR  R   R  R  t   classmethodR  (    (    (    s0   lib/python2.7/site-packages/distributed/utils.pyR    s   			c          C` sK   xD t  j j j j   D]- }  x$ t  j |   j D] } | j   q/ Wq Wd S(   so    Python 2's logger's locks don't survive a fork event

    https://github.com/dask/distributed/issues/1491
    N(   R   t   Loggert   managert
   loggerDictt   keysR   R   t
   createLock(   R   R  (    (    s0   lib/python2.7/site-packages/distributed/utils.pyt   reset_logger_locks  s    t   asyncioi   t   notebook(   t   NotebookAppc         C` sQ   t  r | t j |   j k St j |   r7 |  j }  n  | t j |   j k Sd  S(   N(	   R   t   inspectt	   signaturet
   parametersR   t   is_coroutine_functionR7   R   R5   (   R.   t   keyword(    (    s0   lib/python2.7/site-packages/distributed/utils.pyt   has_keyword  s
    i  s   #440154s   #471669s   #472A79s   #433C84s   #3C4D8As   #355D8Cs   #2E6C8Es   #287A8Es   #23898Ds   #1E978As   #20A585s   #2EB27Cs   #45BF6Fs   #64CB5Ds   #88D547s   #AFDC2Es   #D7E219s   #FDE724c         C` sC   t  t |   j    } t | j   d  d  } | | t |  S(   Ni   i   (   R   R2   RL  RT  t	   hexdigestR   (   R   t   paletteR  R  (    (    s0   lib/python2.7/site-packages/distributed/utils.pyt   color_of  s    c         C` s9   t  j |   r t St j d k r5 t j |   r5 t St S(   Ni   i   (   i   i   (   R   R  R4   R   R  R  t   iscoroutinefunctionR8   (   R   (    (    s0   lib/python2.7/site-packages/distributed/utils.pyR    s
    c         c` sG   t    } d  Vt    } | | t |   k rC t j | d d n  d  S(   Nt
   stackleveli   (   R   R  RE   RF   (   R  t   msgR   R   (    (    s0   lib/python2.7/site-packages/distributed/utils.pyt   warn_on_duration  s
    		c         C` s5   y |  j  d |  j SWn t k
 r0 t |   SXd S(   s    Return name of type

    Examples
    --------
    >>> from distributed import Scheduler
    >>> typename(Scheduler)
    'distributed.scheduler.Scheduler'
    R   N(   R   R0   R1   R2   (   R   (    (    s0   lib/python2.7/site-packages/distributed/utils.pyt   typename  s    	(    (    (   i   (    i  i@B i ʚ;I    I Ƥ~ i   i   i   @I       I       iQ i  i  iQ (   t
   __future__R    R   R   t   atexitt   collectionsR   t
   contextlibR   t   datetimeR   R  t   hashlibR   R  t   jsonR   R%   t   numbersR   R  R   R   RF  R@   R   R   t	   importlibR	   R   R   R   RE   R   R   t   tblib.pickling_supportt   tblibt   compatibilityR
   R   R   R   R;   t   ImportErrorR<   R"   R   t   toolzR  R   t   tornado.ioloopR   R   R   R   R   R   R   t   metricst   dask.contextR   t   localR   R0   R   t   _loggert
   no_defaultR-   t
   mp_contextR3   R:   R?   t   memoizeRP   RR   RT   R[   R]   Rn   Rf   Rq   Rr   R   t   objectR   R   R   R   R   t   compileR   R   R   R8   R   R   R  R  t   pickling_supportt   installR  R  R  R   R$  R'  R)  R*  R,  R/  R2  R>  RB  RK  RQ  RZ  R[  R/   R\  R]  t   _shutting_downR_  t   registerR   Rb  Rp  Rq  Ru  R  R   t   updateR  R  t   tds2R  R  R   RO  RP  R  R  R  R  R  R  t   HandlerR  R  R'   t   jupyter_event_loop_initializedt	   traitletst   notebook.notebookappR  R#   t   Applicationt   initializedR   t   instanceR  t   tornado.platform.asynciot   set_event_loop_policyR   t   AnyThreadEventLoopPolicyR  R  R  R  R  R  (    (    (    s0   lib/python2.7/site-packages/distributed/utils.pyt   <module>   s  "

(			
		
	)(	5			6.	*				
					!4						,	
	*

(				"	
			