
\c           @` s8  d  d l  m Z m Z m Z d  d l m Z m Z m Z d  d l m	 Z	 d  d l
 m Z d  d l Z d  d l Z d  d l Z d  d l m Z d  d l Z d  d l Z d  d l Z d  d l Z d  d l Z d  d l Z d  d l Z d  d l Z y, d  d l m Z m Z m Z m Z m Z Wn9 e k
 rSd  d l  m Z m Z m Z m Z m Z n Xd  d l  m! Z! m" Z" m# Z# m$ Z$ d  d	 l% m& Z& d  d
 l' m( Z( d  d l) m* Z* d  d l+ Z+ d d l, m- Z- d d l. m/ Z/ m0 Z0 m1 Z1 m2 Z2 d d l3 m4 Z4 m5 Z5 m6 Z6 m7 Z7 d d l8 m9 Z9 m: Z: m; Z; m< Z< m= Z= d d l> m? Z? d d l@ mA ZA d d lB mC ZC d d lD mE ZE d d lF mG ZG d d lH mI ZI mJ ZJ mK ZK mL ZL mM ZM mN ZN mO ZO mP ZP mQ ZQ mR ZR mS ZS mT ZT d d lU mV ZV mW ZW d d lX mY ZY mZ ZZ d d l[ m\ Z\ d d l] m^ Z^ d d l_ m` Z` d d la mb Zb d d lc md Zd d d le mf Zf d d lg mh Zh e ji ej  Zk e+ jl jm d   Zn e+ jl jm d!  Zo e+ jl jm d"  Zp e+ jl jm d#  Zq eb e\ e` e^ eh ed g Zr e+ jl jm d$  rer js ef  n  d% d& d' d( d) d* h Zt d+ eu f d,     YZv d- eu f d.     YZw d/ eu f d0     YZx d1 e6 f d2     YZy d3 ey f d4     YZz d5 e7 f d6     YZ{ d7   Z| d8   Z} d9   Z~ d:   Z d;   Z d< eC f d=     YZ d>   Z d?   Z d@   Z dA   Z d  g Z dB dC h Z dD   Z dE e f dF     YZ d S(G   i    (   t   print_functiont   divisiont   absolute_import(   t   defaultdictt   dequet   OrderedDict(   t	   timedelta(   t   partialN(   t   Number(   t   frequenciest   merget   pluckt   merge_sortedt   first(   t   valmapt   secondt   composet   groupby(   t   gen(   t   Return(   t   IOLoopi   (   t   BatchedSend(   t   normalize_addresst   resolve_addresst   get_address_hostt   unparse_host_port(   t   finalizet   unicodet   Mappingt   Set(   t   rpct   connectt	   send_recvt   clean_exceptiont   CommClosedError(   t   profile(   t   time(   t
   ServerNode(   t   setproctitle(   t   Security(   t   Allt   ignoringt   get_ipt   get_fileno_limitt
   log_errorst	   key_splitt   validate_keyt
   no_defaultt   DequeHandlert   parse_timedeltat   PeriodicCallbackt   shutting_down(   t   scatter_to_workerst   gather_from_workers(   t   enable_gc_diagnosist   disable_gc_diagnosis(   t   PublishExtension(   t   QueueExtension(   t   ReplayExceptionScheduler(   t   LockExtension(   t   PubSubSchedulerExtension(   t   WorkStealing(   t   VariableExtensions   distributed.scheduler.bandwidths&   distributed.scheduler.allowed-failuress   distributed.admin.pdb-on-errs'   distributed.scheduler.default-data-sizes#   distributed.scheduler.work-stealingt   releasedt   waitings	   no-workert
   processingt   erredt   memoryt   ClientStatec           B` s/   e  Z d  Z d Z d   Z d   Z d   Z RS(   sx  
    A simple object holding information about a client.

    .. attribute:: client_key: str

       A unique identifier for this client.  This is generally an opaque
       string generated by the client itself.

    .. attribute:: wants_what: {TaskState}

       A set of tasks this client wants kept in memory, so that it can
       download its result when desired.  This is the reverse mapping of
       :class:`TaskState.who_wants`.

       Tasks are typically removed from this set when the corresponding
       object in the client's space (for example a ``Future`` or a Dask
       collection) gets garbage-collected.

    t
   client_keyt
   wants_whatt	   last_seenc         C` s%   | |  _  t   |  _ t   |  _ d  S(   N(   RE   t   setRF   R$   RG   (   t   selft   client(    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyt   __init__v   s    	c         C` s   d |  j  f S(   Ns   <Client %r>(   RE   (   RI   (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyt   __repr__{   s    c         C` s   |  j  S(   N(   RE   (   RI   (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyt   __str__~   s    (   RE   RF   RG   (   t   __name__t
   __module__t   __doc__t	   __slots__RK   RL   RM   (    (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyRD   _   s
   		t   WorkerStatec           B` se   e  Z d  Z d Z d d d d d d d d  Z e d    Z d   Z d   Z	 d   Z
 d   Z RS(   s
  
    A simple object holding information about a worker.

    .. attribute:: address

       This worker's unique key.  This can be its connected address
       (such as ``'tcp://127.0.0.1:8891'``) or an alias (such as ``'alice'``).

    .. attribute:: processing: {TaskState: cost}

       A dictionary of tasks that have been submitted to this worker.
       Each task state is asssociated with the expected cost in seconds
       of running that task, summing both the task's expected computation
       time and the expected communication time of its result.

       Multiple tasks may be submitted to a worker in advance and the worker
       will run them eventually, depending on its execution resources
       (but see :doc:`work-stealing`).

       All the tasks here are in the "processing" state.

       This attribute is kept in sync with :attr:`TaskState.processing_on`.

    .. attribute:: has_what: {TaskState}

       The set of tasks which currently reside on this worker.
       All the tasks here are in the "memory" state.

       This is the reverse mapping of :class:`TaskState.who_has`.

    .. attribute:: nbytes: int

       The total memory size, in bytes, used by the tasks this worker
       holds in memory (i.e. the tasks in this worker's :attr:`has_what`).

    .. attribute:: ncores: int

       The number of CPU cores made available on this worker.

    .. attribute:: resources: {str: Number}

       The available resources on this worker like ``{'gpu': 2}``.
       These are abstract quantities that constrain certain tasks from
       running at the same time on this worker.

    .. attribute:: used_resources: {str: Number}

       The sum of each resource used by all tasks allocated to this worker.
       The numbers in this dictionary can only be less or equal than
       those in this worker's :attr:`resources`.

    .. attribute:: occupancy: Number

       The total expected runtime, in seconds, of all tasks currently
       processing on this worker.  This is the sum of all the costs in
       this worker's :attr:`processing` dictionary.

    .. attribute:: status: str

       The current status of the worker, either ``'running'`` or ``'closed'``

    .. attribute:: last_seen: Number

       The last time we received a heartbeat from this worker, in local
       scheduler time.

    .. attribute:: actors: {TaskState}

       A set of all TaskStates on this worker that are actors.  This only
       includes those actors whose state actually lives on this worker, not
       actors to which this worker has a reference.

    t   actorst   addresst   has_whatRG   t   local_directoryt   memory_limitt   metricst   namet   nbytest   ncorest	   occupancyt   pidRA   t	   resourcest   servicest   statust
   time_delayt   used_resourcesi    c         C` s   | |  _  | |  _ | |  _ | |  _ | |  _ | |  _ | p? i  |  _ d |  _ d |  _ d |  _	 i  |  _
 d |  _ d |  _ t   |  _ t   |  _ i  |  _ i  |  _ i  |  _ d  S(   Nt   runningi    (   RT   R]   RY   R[   RW   RV   R_   R`   RZ   R\   RX   RG   Ra   RH   RS   RU   RA   R^   Rb   (   RI   RT   R]   RY   R[   RW   RV   R_   (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyRK      s$    
														c         C` s   t  |  j  S(   N(   R   RT   (   RI   (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyt   host  s    c         C` sb   t  d |  j d |  j d |  j d |  j d |  j d |  j d |  j  } d   |  j D | _ | S(	   sG    Return a version of this object that is appropriate for serialization RT   R]   RY   R[   RW   RV   R_   c         S` s   h  |  ] } | j   q S(    (   t   key(   t   .0t   ts(    (    s4   lib/python2.7/site-packages/distributed/scheduler.pys	   <setcomp>  s   	 (	   RR   RT   R]   RY   R[   RW   RV   R_   RA   (   RI   t   ws(    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyt   clean  s    						c         C` s&   d |  j  t |  j  t |  j  f S(   Ns'   <Worker %r, memory: %d, processing: %d>(   RT   t   lenRU   RA   (   RI   (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyRL     s    c         C` s   |  j  S(   N(   RT   (   RI   (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyRM     s    c         C` so   i d d 6|  j  d 6|  j d 6|  j d 6|  j d 6|  j  d 6|  j d 6|  j d	 6|  j d
 6|  j d 6|  j d 6S(   Nt   Workert   typet   idRd   R^   RV   RY   R[   RW   RG   R_   RX   (	   RY   Rd   R^   RV   R[   RW   RG   R_   RX   (   RI   (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyt   identity  s    








(   RS   RT   RU   RG   RV   RW   RX   RY   RZ   R[   R\   R]   RA   R^   R_   R`   Ra   Rb   N(   RN   RO   RP   RQ   t   NoneRK   t   propertyRd   Ri   RL   RM   Rn   (    (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyRR      s>   I                 			t	   TaskStatec           B` sA   e  Z d  Z d Z d   Z d   Z d   Z d   Z d   Z RS(    s%  
    A simple object holding information about a task.

    .. attribute:: key: str

       The key is the unique identifier of a task, generally formed
       from the name of the function, followed by a hash of the function
       and arguments, like ``'inc-ab31c010444977004d656610d2d421ec'``.

    .. attribute:: prefix: str

       The key prefix, used in certain calculations to get an estimate
       of the task's duration based on the duration of other tasks in the
       same "family" (for example ``'inc'``).

    .. attribute:: run_spec: object

       A specification of how to run the task.  The type and meaning of this
       value is opaque to the scheduler, as it is only interpreted by the
       worker to which the task is sent for executing.

       As a special case, this attribute may also be ``None``, in which case
       the task is "pure data" (such as, for example, a piece of data loaded
       in the scheduler using :meth:`Client.scatter`).  A "pure data" task
       cannot be computed again if its value is lost.

    .. attribute:: priority: tuple

       The priority provides each task with a relative ranking which is used
       to break ties when many tasks are being considered for execution.

       This ranking is generally a 2-item tuple.  The first (and dominant)
       item corresponds to when it was submitted.  Generally, earlier tasks
       take precedence.  The second item is determined by the client, and is
       a way to prioritize tasks within a large graph that may be important,
       such as if they are on the critical path, or good to run in order to
       release many dependencies.  This is explained further in
       :doc:`Scheduling Policy <scheduling-policies>`.

    .. attribute:: state: str

       This task's current state.  Valid states include ``released``,
       ``waiting``, ``no-worker``, ``processing``, ``memory``, ``erred``
       and ``forgotten``.  If it is ``forgotten``, the task isn't stored
       in the ``tasks`` dictionary anymore and will probably disappear
       soon from memory.

    .. attribute:: dependencies: {TaskState}

       The set of tasks this task depends on for proper execution.  Only
       tasks still alive are listed in this set.  If, for whatever reason,
       this task also depends on a forgotten task, the
       :attr:`has_lost_dependencies` flag is set.

       A task can only be executed once all its dependencies have already
       been successfully executed and have their result stored on at least
       one worker.  This is tracked by progressively draining the
       :attr:`waiting_on` set.

    .. attribute:: dependents: {TaskState}

       The set of tasks which depend on this task.  Only tasks still alive
       are listed in this set.

       This is the reverse mapping of :attr:`dependencies`.

    .. attribute:: has_lost_dependencies: bool

       Whether any of the dependencies of this task has been forgotten.
       For memory consumption reasons, forgotten tasks are not kept in
       memory even though they may have dependent tasks.  When a task is
       forgotten, therefore, each of its dependents has their
       :attr:`has_lost_dependencies` attribute set to ``True``.

       If :attr:`has_lost_dependencies` is true, this task cannot go
       into the "processing" state anymore.

    .. attribute:: waiting_on: {TaskState}

       The set of tasks this task is waiting on *before* it can be executed.
       This is always a subset of :attr:`dependencies`.  Each time one of the
       dependencies has finished processing, it is removed from the
       :attr:`waiting_on` set.

       Once :attr:`waiting_on` becomes empty, this task can move from the
       "waiting" state to the "processing" state (unless one of the
       dependencies errored out, in which case this task is instead
       marked "erred").

    .. attribute:: waiters: {TaskState}

       The set of tasks which need this task to remain alive.  This is always
       a subset of :attr:`dependents`.  Each time one of the dependents
       has finished processing, it is removed from the :attr:`waiters`
       set.

       Once both :attr:`waiters` and :attr:`who_wants` become empty, this
       task can be released (if it has a non-empty :attr:`run_spec`) or
       forgotten (otherwise) by the scheduler, and by any workers
       in :attr:`who_has`.

       .. note:: Counter-intuitively, :attr:`waiting_on` and
          :attr:`waiters` are not reverse mappings of each other.

    .. attribute:: who_wants: {ClientState}

       The set of clients who want this task's result to remain alive.
       This is the reverse mapping of :attr:`ClientState.wants_what`.

       When a client submits a graph to the scheduler it also specifies
       which output tasks it desires, such that their results are not released
       from memory.

       Once a task has finished executing (i.e. moves into the "memory"
       or "erred" state), the clients in :attr:`who_wants` are notified.

       Once both :attr:`waiters` and :attr:`who_wants` become empty, this
       task can be released (if it has a non-empty :attr:`run_spec`) or
       forgotten (otherwise) by the scheduler, and by any workers
       in :attr:`who_has`.

    .. attribute:: who_has: {WorkerState}

       The set of workers who have this task's result in memory.
       It is non-empty iff the task is in the "memory" state.  There can be
       more than one worker in this set if, for example, :meth:`Client.scatter`
       or :meth:`Client.replicate` was used.

       This is the reverse mapping of :attr:`WorkerState.has_what`.

    .. attribute:: processing_on: WorkerState (or None)

       If this task is in the "processing" state, which worker is currently
       processing it.  Otherwise this is ``None``.

       This attribute is kept in sync with :attr:`WorkerState.processing`.

    .. attribute:: retries: int

       The number of times this task can automatically be retried in case
       of failure.  If a task fails executing (the worker returns with
       an error), its :attr:`retries` attribute is checked.  If it is
       equal to 0, the task is marked "erred".  If it is greater than 0,
       the :attr:`retries` attribute is decremented and execution is
       attempted again.

    .. attribute:: nbytes: int (or None)

       The number of bytes, as determined by ``sizeof``, of the result
       of a finished task.  This number is used for diagnostics and to
       help prioritize work.

    .. attribute:: type: str

       The type of the object as a string.  Only present for tasks that have
       been computed.

    .. attribute:: exception: object

       If this task failed executing, the exception object is stored here.
       Otherwise this is ``None``.

    .. attribute:: traceback: object

       If this task failed executing, the traceback object is stored here.
       Otherwise this is ``None``.

    .. attribute:: exception_blame: TaskState (or None)

       If this task or one of its dependencies failed executing, the
       failed task is stored here (possibly itself).  Otherwise this
       is ``None``.

    .. attribute:: suspicious: int

       The number of times this task has been involved in a worker death.

       Some tasks may cause workers to die (such as calling ``os._exit(0)``).
       When a worker dies, all of the tasks on that worker are reassigned
       to others.  This combination of behaviors can cause a bad task to
       catastrophically destroy all workers on the cluster, one after
       another.  Whenever a worker dies, we mark each task currently
       processing on that worker (as recorded by
       :attr:`WorkerState.processing`) as suspicious.

       If a task is involved in three deaths (or some other fixed constant)
       then we mark the task as ``erred``.

    .. attribute:: host_restrictions: {hostnames}

       A set of hostnames where this task can be run (or ``None`` if empty).
       Usually this is empty unless the task has been specifically restricted
       to only run on certain hosts.  A hostname may correspond to one or
       several connected workers.

    .. attribute:: worker_restrictions: {worker addresses}

       A set of complete worker addresses where this can be run (or ``None``
       if empty).  Usually this is empty unless the task has been specifically
       restricted to only run on certain workers.

       Note this is tracking worker addresses, not worker states, since
       the specific workers may not be connected at this time.

    .. attribute:: resource_restrictions: {resource: quantity}

       Resources required by this task, such as ``{'gpu': 1}`` or
       ``{'memory': 1e9}`` (or ``None`` if empty).  These are user-defined
       names and are matched against the contents of each
       :attr:`WorkerState.resources` dictionary.

    .. attribute:: loose_restrictions: bool

       If ``False``, each of :attr:`host_restrictions`,
       :attr:`worker_restrictions` and :attr:`resource_restrictions` is
       a hard constraint: if no worker is available satisfying those
       restrictions, the task cannot go into the "processing" state and
       will instead go into the "no-worker" state.

       If ``True``, the above restrictions are mere preferences: if no worker
       is available satisfying those restrictions, the task can still go
       into the "processing" state and be sent for execution to another
       connected worker.

    .. attribute: actor: bool

       Whether or not this task is an Actor.
    t   actorRe   t   prefixt   run_spect   dependenciest
   dependentst   priorityt   host_restrictionst   worker_restrictionst   resource_restrictionst   loose_restrictionst   statet   has_lost_dependenciest
   waiting_ont   waiterst   processing_ont   who_hast	   who_wantst	   exceptiont	   tracebackt   exception_blamet
   suspicioust   retriesRZ   Rl   c         C` s   | |  _  t |  |  _ | |  _ d  |  _ d  |  _ |  _ |  _ d |  _	 |  _
 d  |  _ d  |  _ t   |  _ t   |  _ t   |  _ t   |  _ t   |  _ t   |  _ d  |  _ t |  _ d  |  _ d  |  _ d  |  _ t |  _ d  |  _ d  |  _ d  S(   Ni    (   Re   R-   Rs   Rt   Ro   R|   R   R   R   R   R   RZ   Rw   RH   R   Ru   Rv   R~   R   R   R   t   FalseR}   Rx   Ry   Rz   R{   Rr   Rl   (   RI   Re   Rt   (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyRK   A  s,    												c         C` s   |  j  } | d  k	 r | St S(   N(   RZ   Ro   t   DEFAULT_DATA_SIZE(   RI   RZ   (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyt
   get_nbytesY  s    	c         C` sI   |  j  } | | p d } x  |  j D] } | j  | 7_  q# W| |  _  d  S(   Ni    (   RZ   R   (   RI   RZ   t
   old_nbytest   diffRh   (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyt
   set_nbytes]  s
    	c         C` s   d |  j  |  j f S(   Ns   <Task %r %s>(   Re   R|   (   RI   (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyRL   d  s    c         C` sL  yx; |  j  D]0 } t | t  s t t |  |  j  f   q Wx; |  j D]0 } t | t  sK t t |  |  j f   qK Wx; |  j D]0 } t | t  s t t |  |  j f   q Wx; |  j	 D]0 } t | t  s t t |  |  j	 f   q Wt
 |   Wn? t k
 rG} t j |  t rHd d  l } | j   qHn Xd  S(   Ni    (   R   t
   isinstanceRD   t   AssertionErrort   reprR   RR   Ru   Rq   Rv   t   validate_task_statet	   Exceptiont   loggerR   t   LOG_PDBt   pdbt	   set_trace(   RI   t   csRh   Rg   t   eR   (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyt   validateg  s    ....(   Rr   Re   Rs   Rt   Ru   Rv   Rw   Rx   Ry   Rz   R{   R|   R}   R~   R   R   R   R   R   R   R   R   R   RZ   Rl   (	   RN   RO   RP   RQ   RK   R   R   RL   R   (    (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyRq   /  s>                           				t   _StateLegacyMappingc           B` s;   e  Z d  Z d   Z d   Z d   Z d   Z d   Z RS(   sP   
    A mapping interface mimicking the former Scheduler state dictionaries.
    c         C` s   | |  _  | |  _ d  S(   N(   t   _statest	   _accessor(   RI   t   statest   accessor(    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyRK     s    	c         C` s   t  |  j  S(   N(   t   iterR   (   RI   (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyt   __iter__  s    c         C` s   t  |  j  S(   N(   Rj   R   (   RI   (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyt   __len__  s    c         C` s   |  j  |  j |  S(   N(   R   R   (   RI   Re   (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyt   __getitem__  s    c         C` s   d |  j  t |   f S(   Ns   %s(%s)(   t	   __class__t   dict(   RI   (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyRL     s    (   RN   RO   RP   RK   R   R   R   RL   (    (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyR   z  s   				t   _OptionalStateLegacyMappingc           B` s)   e  Z d  Z d   Z d   Z d   Z RS(   sb   
    Similar to _StateLegacyMapping, but a false-y value is interpreted
    as a missing key.
    c         c` sA   |  j  } x1 |  j j   D]  \ } } | |  r | Vq q Wd  S(   N(   R   R   t   items(   RI   R   t   kt   v(    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyR     s    	c         ` s,   |  j    t   f d   |  j j   D  S(   Nc         3` s!   |  ] } t    |   Vq d  S(   N(   t   bool(   Rf   R   (   R   (    s4   lib/python2.7/site-packages/distributed/scheduler.pys	   <genexpr>  s    (   R   t   sumR   t   values(   RI   (    (   R   s4   lib/python2.7/site-packages/distributed/scheduler.pyR     s    	c         C` s*   |  j  |  j |  } | r  | St  d  S(   N(   R   R   t   KeyError(   RI   Re   R   (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyR     s    (   RN   RO   RP   R   R   R   (    (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyR     s   		t   _StateLegacySetc           B` s;   e  Z d  Z d   Z d   Z d   Z d   Z d   Z RS(   sh   
    Similar to _StateLegacyMapping, but exposes a set containing
    all values with a true value.
    c         C` s   | |  _  | |  _ d  S(   N(   R   R   (   RI   R   R   (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyRK     s    	c         ` s     f d     j  j   D S(   Nc         3` s*   |  ]  \ } }   j  |  r | Vq d  S(   N(   R   (   Rf   R   R   (   RI   (    s4   lib/python2.7/site-packages/distributed/scheduler.pys	   <genexpr>  s    (   R   R   (   RI   (    (   RI   s4   lib/python2.7/site-packages/distributed/scheduler.pyR     s    c         C` s(   t  t t t |  j |  j j      S(   N(   R   t   mapR   R   R   R   (   RI   (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyR     s    c         C` s1   |  j  j |  } | d  k	 o0 t |  j |   S(   N(   R   t   getRo   R   R   (   RI   R   t   st(    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyt   __contains__  s    c         C` s   d |  j  t |   f S(   Ns   %s(%s)(   R   RH   (   RI   (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyRL     s    (   RN   RO   RP   RK   R   R   R   RL   (    (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyR     s   				c         C` s   d   |  D S(   sA   
    Transform a set of task states into a set of task keys.
    c         S` s   h  |  ] } | j   q S(    (   Re   (   Rf   Rg   (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pys	   <setcomp>  s   	 (    (   t   tasks(    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyt   _legacy_task_key_set  s    c         C` s   d   |  D S(   sE   
    Transform a set of client states into a set of client keys.
    c         S` s   h  |  ] } | j   q S(    (   RE   (   Rf   R   (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pys	   <setcomp>  s   	 (    (   t   clients(    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyt   _legacy_client_key_set  s    c         C` s   d   |  D S(   sE   
    Transform a set of worker states into a set of worker keys.
    c         S` s   h  |  ] } | j   q S(    (   RT   (   Rf   Rh   (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pys	   <setcomp>  s   	 (    (   t   workers(    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyt   _legacy_worker_key_set  s    c         C` s   d   |  j    D S(   sS   
    Transform a dict of {task state: value} into a dict of {task key: value}.
    c         S` s"   i  |  ] \ } } | | j   q S(    (   Re   (   Rf   Rg   t   value(    (    s4   lib/python2.7/site-packages/distributed/scheduler.pys
   <dictcomp>  s   	 (   R   (   t	   task_dict(    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyt   _legacy_task_key_dict  s    c         C` s   |  d  k	 r |  j Sd  S(   N(   Ro   Re   (   t   task(    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyt   _task_key_or_none  s    t	   Schedulerc           B` s  e  Z d  Z d Z e d d e e e e e e e e d  Z d   Z e d  Z	 e d  Z
 d   Z d	   Z d e d
  Z e j d    Z e j e e e d   Z e j e e e d   Z d   Z e j e e e e e e e d   Z e j e e f  e e e e e e e e e e d e e d   Z e e e e e e e e e e d e d d  Z e e d  Z e e e e d  Z e e e e d  Z e e e d  Z e e e e d  Z e e e d  Z d e d  Z e e d  Z e e d  Z  e d  Z! d   Z" d   Z# d    Z$ d!   Z% d"   Z& d#   Z' e d$  Z( e d%  Z) e e d&  Z* e j e d'   Z+ e d(  Z, d)   Z- d*   Z. e e d+  Z/ e d,  Z0 e e e d-  Z1 e e d.  Z2 e e e d/  Z3 e e e d0  Z4 e j e e d1   Z5 e e d2  Z6 d3   Z7 d4   Z8 e j e e e e e d5 d6   Z9 e j e e e d7   Z: d8   Z; e j e d9 d:   Z< e j e e e e e e d;   Z= e j e e e e d<   Z> e j e e e d=   Z? e j e e e e d5 e d>   Z@ e e e e d?  ZA e j e e e e d@   ZB e e f  dA  ZC e e e e e dB  ZD e e e dC  ZE e j e e e dD dE   ZF e e dF  ZG e e dG  ZH e e dH  ZI e e dI  ZJ e j e e dJ   ZK e e e dK  ZL dL   ZM dM dN  ZN f  i  e dO  ZO e e e dP  ZP e e eQ dQ  ZR e e dR  ZS e e e e dS  ZT e j e dT   ZU e dU  ZV e e dV  ZW dW   ZX dX   ZY dY   ZZ dZ   Z[ e e d[  Z\ e e e e e d\  Z] e d]  Z^ d^   Z_ d_   Z` d`   Za da   Zb e e e db  Zc dc   Zd dd   Ze de   Zf df   Zg dg   Zh dh   Zi di   Zj dj   Zk ek Zl e e dk  Zm e dl  Zn dm   Zo dn   Zp do   Zq e e e dp  Zr dq   Zs e dr  Zt ds   Zu dt   Zv e du  Zw dv   Zx e j e e e e e e dw   Zy e j e e e e e e dx   Zz e e dy  Z{ e j e e e dz   Z| d d{  Z} d|   Z~ d}   Z d~   Z RS(   s
   Dynamic distributed task scheduler

    The scheduler tracks the current state of workers, data, and computations.
    The scheduler listens for events and responds by controlling workers
    appropriately.  It continuously tries to use the workers to execute an ever
    growing dask graph.

    All events are handled quickly, in linear time with respect to their input
    (which is often of constant size) and generally within a millisecond.  To
    accomplish this the scheduler tracks a lot of state.  Every operation
    maintains the consistency of this state.

    The scheduler communicates with the outside world through Comm objects.
    It maintains a consistent and valid view of the world even when listening
    to several clients at once.

    A Scheduler is typically started either with the ``dask-scheduler``
    executable::

         $ dask-scheduler
         Scheduler started at 127.0.0.1:8786

    Or within a LocalCluster a Client starts up without connection
    information::

        >>> c = Client()  # doctest: +SKIP
        >>> c.cluster.scheduler  # doctest: +SKIP
        Scheduler(...)

    Users typically do not interact with the scheduler directly but rather with
    the client object ``Client``.

    **State**

    The scheduler contains the following state variables.  Each variable is
    listed along with what it stores and a brief description.

    * **tasks:** ``{task key: TaskState}``
        Tasks currently known to the scheduler
    * **unrunnable:** ``{TaskState}``
        Tasks in the "no-worker" state

    * **workers:** ``{worker key: WorkerState}``
        Workers currently connected to the scheduler
    * **idle:** ``{WorkerState}``:
        Set of workers that are not fully utilized
    * **saturated:** ``{WorkerState}``:
        Set of workers that are not over-utilized

    * **host_info:** ``{hostname: dict}``:
        Information about each worker host

    * **clients:** ``{client key: ClientState}``
        Clients currently connected to the scheduler

    * **services:** ``{str: port}``:
        Other services running on this scheduler, like Bokeh
    * **loop:** ``IOLoop``:
        The running Tornado IOLoop
    * **client_comms:** ``{client key: Comm}``
        For each client, a Comm object used to receive task requests and
        report task status updates.
    * **stream_comms:** ``{worker key: Comm}``
        For each worker, a Comm object from which we both accept stimuli and
        report results
    * **task_duration:** ``{key-prefix: time}``
        Time we expect certain functions to take, e.g. ``{'sum': 0.25}``
    * **coroutines:** ``[Futures]``:
        A list of active futures that control operation
    iR"  t   500mst   60sc         K` s|	  |  j    | |  _ | |  _ d  |  _ t j   |  _ t | d d |  _	 t | d d |  _
 d  |  _ | pp i  |  _ i  |  _ | |  _ |
 p t j j d  }
 |
 r t |
  n d  |  _ | p t j j d  } | r t |  |  _ n	 d  |  _ t   |  _ |	 pt   |  _ t |  j t  s*t  |  j j d  |  _ |  j j d  |  _ | pct j   |  _  t!   |  _" t!   |  _# g  |  _$ g  |  _% d  |  _& t!   |  _' x~ do d d t( f d d t( f dp g D]X \ } } } t) j* |  } | d  k	 r	t+ | |  } n  t, |  | t- |  j' |   qWx dq d d t. f d d t/ f d d t( f d d t( f dr ds dt du dv dw dx d d t0 f g D]X \ } } } t) j* |  } | d  k	 rt+ | |  } n  t, |  | t1 |  j' |   qWxc dy g D]X \ } } } t) j* |  } | d  k	 r&t+ | |  } n  t, |  | t2 |  j' |   qWd |  _3 d  |  _4 d |  _5 t6   |  _7 d |  _8 t!   |  _9 t!   |  _: d    t; D |  _< t= t6  |  _> t!   |  _? xl d! d! t( f g D]X \ } } } t) j* |  } | d  k	 rt+ | |  } n  t, |  | t- |  j? |   qWt@ d"  |  j? d" <tA jB   |  _C x dz d{ d| d} d~ d d+ d+ tD f d, d, t( f g D]X \ } } } t) j* |  } | d  k	 rt+ | |  } n  t, |  | t- |  jC |   q~WtA jE d- t) j* d.   |  _F t6   |  _G d |  _H d |  _I t= t!  |  _J t= t!  |  _K t!   |  _L |  j7 g |  _M |  jC |  jJ |  jK |  jL g |  _N i  |  _O g  |  _P tQ d/ t j j d0   |  _R tQ d/ t j j d0   |  _S g  |  _T i |  jU d1 6|  jV d2 6|  jW d3 6|  jX d4 6|  jY d5 6|  jZ d6 6|  j[ d7 6|  j\ d8 6} i |  j] d9 6|  j^ d: 6|  j_ d; 6|  j` d< 6|  ja d= 6|  jb d> 6|  jc d? 6|  jd d@ 6} i# |  je dA 6|  jf dB 6|  jg dC 6|  jh dD 6|  ji dE 6|  jj dF 6|  jk dG 6|  jl dH 6|  jm dI 6|  jn dJ 6|  jo dK 6|  jp d# 6|  jq d, 6|  jr d 6|  js d+ 6|  jt dL 6|  ju dM 6|  jv dN 6|  jw dO 6|  jx d
 6|  jy dP 6|  jY dQ 6|  jz dR 6|  j{ dS 6|  j| dT 6|  j} dU 6|  j_ dV 6|  j~ dW 6|  j dX 6|  j dY 6|  j dZ 6|  j d[ 6|  j d\ 6|  j d] 6|  j d^ 6|  _ i |  j d 6|  j d 6|  j d 6|  j d 6|  j d 6|  j d 6|  j d 6|  j d 6|  j d 6|  j d 6|  j d 6|  j d 6|  j d 6|  j d 6|  j d 6|  _ t   dd } t t |   j de |  j df t | |  dg |  j  dh | di t dj |  j |  |  j r	t |  j |  j dg | } | |  j dk <n  |  j r>	t |  j |  j dl dg | } | |  j dm <n  | d  k rS	t } n  x | D] } | |   qZ	Wt dn  d  S(   Nt   defaultt   mss    distributed.scheduler.worker-ttls"   distributed.scheduler.idle-timeoutt	   schedulerRw   Ru   Rv   R   RZ   R   R   R@   R~   t   waiting_dataR   t   rprocessingR   Rx   Ry   Rz   t   suspicious_tasksR   t
   exceptionsR   t
   tracebacksR   t   exceptions_blameR   R{   i    c         S` s   i  |  ] } d  |  q S(   gh㈵>(    (   Rf   Rs   (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pys
   <dictcomp>  s   	 RF   s   fire-and-forgetR[   t   worker_bytest   worker_resourcesR^   Rb   R\   t   worker_infoRX   RA   RU   Re   RT   t   maxlens+   distributed.scheduler.transition-log-lengths   task-finisheds
   task-erredt   releases   release-worker-datas   add-keyss   missing-datas   long-runningt
   reschedules   update-graphs   client-desires-keyss   update-datas
   report-keys   client-releases-keyss   heartbeat-clients   close-clientt   restarts   register-clientt   scatters   register-workert
   unregistert   gathert   cancelt   retryt   feedt	   terminatet	   broadcastt   proxyt
   call_stackR#   t   logst   worker_logst   versionst   add_keyst	   rebalancet	   replicatet   start_ipythont   run_functiont   update_datat   set_resourcest   retire_workerst   get_metadatat   set_metadatat   heartbeat_workert   get_task_statust   get_task_streamt   register_worker_callbacksR?   RC   RB   s	   no-workert	   forgotteni   t   handlerst   stream_handlerst   io_loopt   connection_limitt   deserializet   connection_argss
   worker-ttli   s   idle-timeouts   dask-scheduler [not started](   Rw   Rw   N(   R   R   N(   RZ   RZ   N(   R   R   N(   Rx   Rx   N(   Ry   Ry   N(   Rz   Rz   N(   R   R   N(   R   R   N(   R   R   N(   R{   R{   N(   R[   R[   N(   R   RZ   N(   R   R^   N(   Rb   Rb   N(   R\   R\   N(   R   RX   N(   R?   R@   (   R@   R?   (   R@   RA   (   R@   RC   (   RA   R?   (   RA   RC   (   RA   RB   (   s	   no-workerR?   (   s	   no-workerR@   (   R?   R   (   RC   R   (   RB   R   (   RB   R?   (   RC   R?   (   R?   RB   (   t   _setup_loggingt   allowed_failuresR   Ro   R`   t   psutilt   Processt   procR1   t   delete_intervalt   synchronize_worker_intervalt   digestst   service_specsR_   t   scheduler_filet   daskt   configR   t
   worker_ttlt   idle_timeoutR$   t   time_startedR'   t   securityR   R   t   get_connection_argsR   t   get_listen_argst   listen_argsR   t   currentt   loopR   t   client_commst   stream_commst
   coroutinest   _worker_coroutinest   _ipython_kernelR   R   t   operatort
   attrgetterR   t   setattrR   R   R   R   R   R   t
   generationt   _last_clientt
   _last_timeRH   t
   unrunnablet   n_taskst   task_metadatat   datasetst
   fast_taskst   task_durationR   t   unknown_durationsR   RD   t   sortedcontainerst
   SortedDictR   R   t	   SortedSett   idlet	   saturatedt   total_ncorest   total_occupancyt	   host_infoR^   t   aliasest   _task_state_collectionst   _worker_collectionst
   extensionst   pluginsR   t   transition_logt   logt   worker_setupst   handle_task_finishedt   handle_task_erredt   handle_release_datat   release_worker_dataR   t   handle_missing_datat   handle_long_runningR   t   update_grapht   client_desires_keysR   t   report_on_keyt   client_releases_keyst   client_heartbeatt   remove_clientR   t
   add_clientR   t
   add_workert   remove_workerR   t   stimulus_cancelt   stimulus_retryR   t   closeR   R   t
   get_ncorest   get_has_whatt   get_who_hast   get_processingt   get_call_stackt   get_profilet   get_logst   get_worker_logsR   R   R   R   R   R   t   add_resourcesR   R   R   R   R   R   R   R   t   transition_released_waitingt   transition_waiting_releasedt   transition_waiting_processingt   transition_waiting_memoryt   transition_processing_releasedt   transition_processing_memoryt   transition_processing_erredt   transition_no_worker_releasedt   transition_no_worker_waitingt   transition_released_forgottent   transition_memory_forgottent   transition_erred_releasedt   transition_memory_releasedt   transition_released_erredt   _transitionsR+   t   superR   RK   R
   R   R2   t   check_worker_ttlt   periodic_callbackst
   check_idlet   DEFAULT_EXTENSIONSR&   (   RI   R  R   R   R_   R   R$  R   R   R  R   R   t   kwargst   old_attrt   new_attrt   wrapt   funct   worker_handlerst   client_handlersR   t   pct   ext(    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyRK   .  s   
											   				  					





























































					c         C` s    d |  j  t |  j  |  j f S(   Ns)   <Scheduler: "%s" processes: %d cores: %d>(   RT   Rj   R   R  (   RI   (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyRL   @  s    c         C` sh   i t  |   j d 6t |  j  d 6|  j d 6d   |  j j   D d 6d   |  j j   D d 6} | S(   s3    Basic information about ourselves and our cluster Rl   Rm   RT   c         S` s"   i  |  ] \ } } | j  |  q S(    (   t   port(   Rf   Re   R   (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pys
   <dictcomp>M  s   	 R_   c         S` s"   i  |  ] } | j    | j  q S(    (   Rn   RT   (   Rf   t   worker(    (    s4   lib/python2.7/site-packages/distributed/scheduler.pys
   <dictcomp>O  s   	 R   (	   Rl   RN   t   strRm   RT   R_   R   R   R   (   RI   t   commt   d(    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyRn   G  s    
c         C` sv   |  j  | } | j j |  } | d k r/ d S| re d i | j j d  d d 6| j d 6| d 6S| j | f Sd S(   s  
        Get the (host, port) address of the named service on the *worker*.
        Returns None if the service doesn't exist.

        Parameters
        ----------
        worker : address
        service_name : str
            Common services include 'bokeh' and 'nanny'
        protocol : boolean
            Whether or not to include a full address with protocol (True)
            or just a (host, port) pair
        s    %(protocol)s://%(host)s:%(port)ds   ://i    t   protocolRd   Ra  N(   R   R_   R   Ro   RT   t   splitRd   (   RI   Rb  t   service_nameRf  Rh   Ra  (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyt   get_worker_service_addrT  s    
	c   	      C` sp  | d k r d } n  xT|  j  j   D]C\ } } d  } t | t  rU | \ } } n d } t | t t f  r | j d  } n  t | t t f  r | d t	 | d  } } n  t | t  r | \ } } n i  } yN | |  d |  j
 | } | j | d  k	 r| n | | f  | |  j | <Wq% t k
 rg} t j d | | f d t |  d	 d
 q% Xq% Wd  S(   Ns   0.0.0.0t    i    t   :i   R   s+   
Could not launch service '%s' on port %s. s   Got the following message:

t
   stackleveli   (   R   R   Ro   R   t   tupleRc  R   Rg  t   listt   intR  t   listenR_   R   t   warningst   warn(	   RI   t   default_listen_ipR   R   t	   listen_ipRa  RX  t   serviceR   (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyt   start_serviceso  s.    	c         C` s(   x! |  j  j   D] } | j   q Wd  S(   N(   R_   R   t   stop(   RI   Ru  (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyt   stop_services  s    c      
   ` sK  t    |  j   t t  # x |  j D] } | j   q( WWd QXx; |  j D]0 } | j   rL | j   } | r| |  q| qL qL W|  j	 d k rt
 | t  r |  j d | f d |  j t   |  _ d } n1 |  j | d |  j t |  j  |  _ |  j } | d k rd } n  t
 | t  r<| j d  r<d } n  |  j |  d |  _	 t j d |  j  x= |  j j   D], \ } } t j d	 | d
 | | j f  quW|  j j |  j  n  |  j r#t |  j d  # }	 t  j! |  j"   |	 d d Wd QX|  j     f d   }
 t# |  |
  n  |  j$   t% d |  j f  |  j&   S(   s8    Clear out old state and restart all running coroutines NRc   Rj  R  s   0.0.0.0s	   inproc://t	   localhosts     Scheduler at: %25ss   %11s at: %25ss   %s:%dt   wt   indenti   c           ` s&   t  j j    r" t  j    n  d  S(   N(   t   ost   patht   existst   remove(    (   t   fn(    s4   lib/python2.7/site-packages/distributed/scheduler.pyt   del_scheduler_file  s    s   dask-scheduler [%s]('   R6   t   clear_task_stateR)   t   AttributeErrorR
  R   R	  t   doneR   R`   R   Ro  Rp  R  R*   t   ipR   t   listen_addressRc  t
   startswithRv  R   t   infoRT   R_   R   Ra  R  t   add_callbackt   reevaluate_occupancyR   t   opent   jsont   dumpRn   R   t   start_periodic_callbacksR&   t   finished(   RI   t   addr_or_portt   start_queuest   ct   cort   excRt  R   R   t   fR  (    (   R  s4   lib/python2.7/site-packages/distributed/scheduler.pyt   start  sJ    
					$	"	
c         c` s2   x+ t  d   |  j D  r- t |  j  Vq Wd S(   s'    Wait until all coroutines have ceased c         s` s   |  ] } | j    Vq d  S(   N(   R  (   Rf   R  (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pys	   <genexpr>  s    N(   t   anyR	  R(   (   RI   (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyR    s    c         c` s<  |  j  j d  r d Sd |  _  t j d  t d  | r x( |  j D] } |  j | i d d 6 qF Wx2 t d  D]! } |  j r t j	 d	  Vqt Pqt Wn  x! |  j
 j   D] } | j   q W|  j
 j   |  j   x. |  j D]# } t t   | j   Wd QXq Wt j d
  g  } x t |  j j    D]p \ }	 } | j   s{| j i d d 6t d 6 | j i d d 6 n  t t   | j | j    Wd QXq1Wx | D] }
 |
 VqW| s|  j   Vn  x! |  j j   D] } | j   qW|  j j   d |  _  |  j   t t  |   j   Vt d  t!   d S(   s    Send cleanup signal to all coroutines then wait until finished

        See Also
        --------
        Scheduler.cleanup
        t   closNt   closings   Scheduler closing...s   dask-scheduler [closing]R:  t   opi   g?s   Scheduler closing all commst   reports   close-streamt   closeds   dask-scheduler [closed]("   R`   R  R   R  R&   R   t   worker_sendt   rangeR   t   sleepRU  R   Rw  t   clearRx  R$  R)   R  t   teardownRn  R  R   R  t   sendR   t   appendR:  R  R  t   abortR   RS  R   R7   (   RI   Rd  t   fastt   close_workersRb  t   iR_  R`  t   futuresRz  t   future(    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyR:    sN    	
	
"		

c      	   C` s   t  j d |  t   t |  j | i d d 6 |  j | d d t } | pR | } |  j | i d d 6t d 6 |  j d	 | d
 |  Wd QXd S(   s    Remove a worker from the cluster

        This both removes the worker from our local state and also sends a
        signal to the worker to shut down.  This works regardless of whether or
        not the worker has a nanny process restarting it
        s   Closing worker %ss   close-workert   actiont   nannyRf  R:  R  R  RT   t   safeN(	   R   R  R,   t	   log_eventRi  t   TrueR  R   R7  (   RI   t   streamRb  R  t
   nanny_addrRT   (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyt   close_worker  s    
c         C` sm   t  d t j j d   |  _ |  j j t j t j j d    t j	 |  j  t
 |  t j |  j  d  S(   Nt   ns   distributed.admin.log-lengths   distributed.admin.log-format(   R0   R   R   R   t   _deque_handlert   setFormattert   loggingt	   FormatterR   t
   addHandlerR   t   removeHandler(   RI   (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyR   (  s    	c         C` sI  |  j  | |  } t |  } t |  } t   }	 | p? t   } | pK i  } | pW i  } |	 |  j | d <|  j j |  }
 |
 s i d d 6St   |
 _ | r | |
 _ n  | r |  j | j	 |  n  t   | } | |
 _
 | r |  j d | d |  n  |  j | t i d d 6|   i d d 6t   d	 6t t |  j   d
 6S(   Ns	   last-seent   missingR`   Rb  R^   t	   heartbeatR  t   OKR$   s   heartbeat-interval(   t   coerce_addressR   R   R$   R   R   R   RG   RX   t   updateRa   RC  R  R
   t   heartbeat_intervalRj   (   RI   Rd  RT   R   t   nowR^   R   RX   Rd   t	   local_nowRh   t   delay(    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyR   6  s2    		 
i    c         c` s  t    |  j | |  } t |  } t |  } |  j j |  } | d k	 re t d |   n  t d | d | d | d | d | d | d |  |  j | <} | |  j	 k r i d	 d
 6d | d 6t
   d 6} | j |  Vd Sd |  j | k r!|  j | j i t   d 6d d 6 n  |  j | d j |  |  j | d c | 7<|  j | 7_ | |  j	 | <|  j d | d | d |	 d |
 d | d |  } |  j |  t d d d |  j  |  j | <| j t | j  k r|  j j |  n  xO |  j D]C } y | j d |  d |  Wqt k
 r@} t j |  qXqW| rxz | D]o } |  j j |  } | d k	 rR| j  d) k rR|  j! | d d | d | | d  | | } |  j" |  qRqRWn  i  } xN t# |  j$  D]= } |  j% |  } | t& k s| | k rd | | j' <qqW| r5|  j" |  n  |  j( | i d! d" 6 |  j( d# i d! d" 6| d 6 t j) d$ t* |   | j i d% d
 6t
   d 6t+ t |  j   d& 6|  j, d' 6 V|  j- d( | d |  VWd QXd S(*   s!    Add a new worker to the cluster s   Worker already exists %sRT   R]   R[   RW   RY   RV   R_   t   errorR`   s   name taken, %st   messageR$   Nt	   addressesi    t   coresR   R  R^   R   RX   t   intervalt   5msR  R   Rb  RA   R@   RC   RZ   t   typenames
   add-workerR  t   alls   Register %sR  s   heartbeat-intervals   worker-setupsRd  (   RA   R@   (.   R,   R  R   R   R   R   Ro   t
   ValueErrorRR   R!  R$   t   writeR   R  RH   t   addR  R   t   check_idle_saturatedR   R  R  R[   Rj   RA   R  R%  R6  R   R   R   R   R|   t
   transitiont   transitionsRn  R  t   valid_workersR  Re   R  R  Rc  R  R(  t   handle_worker(   RI   Rd  RT   t   keysR[   RY   R   RZ   t   typesR  R^   R   RW   RX   R]   R_   RV   Rd   Rh   t   msgt   responset   pluginR   Re   Rg   t   recommendationst   valid(    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyR6  f  s    
(		

c   &      ` s3
  t    } t |  } t |  } t   d k r`  j d | g i d d 6t   d 6 n  x. t   D]  }  | | k rm  | =qm qm W| p i  } d } x t   | k rt   } x t | j    D] \ } } t   f d   | D  r t j	 d |   | =| | =| | k r<| j
 |  n   j i d	 d
 6| d 6d |  j d | g d |  q q Wq WxL | j   D]> \ } } t |  } | | k r| j
 |  n  | | | <qWt   } xU | j   D]G \ } } | r|  j k r j | j d k r| j |  qqW| rlt j j |  } t |  } t |    x | r7| j   }  j | } y | | } Wn t k
 r j | } n Xx | D]w } | | k r| | } n  j | } t   f d   | D  r|  j k r0  j |  | j |  q0qqWqdWx1   D]& }  j | d  | j | d  q?Wn  t |  } t   } g  } x | rO| j   } | | k rqn   j j |  } | d k rt |  j |   }  j | <d | _ n | j s j |  | _ n  | j |  | j |  | j | j | d   qW j d | d |  x | j   D]w \ } }  j j |  } | d k ss| j rqsn  x; | D]3 }  j | } | j j |  | j j |  qWqsWt  t   r f d    D  n  | t! k r1t |  } n  x$ | p=g  D] } t!  j | _" q>W| pmt j# j#   } |	 r j j |	  } | d k	 r| j$ d d } q j% } n@  j& | | k  r j% d 7_%  j% } |  _& n	  j% } xZ t |  | @D]H }  j | } | j$ d k r j | d  | | | f | _$ qqWg  | D] } | j rX| ^ qX}  x; |  D]3 } | j$ d k rz| j rz j% d f | _$ qzqzW| rx | j   D] \ } } | d k rqn   j j |  } | d k rqn  t   | _' t   | _( xU | D]M }! y  j) |!  }! Wn! t* k
 ra| j' j |!  q%X| j( j |!  q%WqW| rx' | D] }  j | } t! | _+ qWqn  | r'xq | j   D]` \ } } | d k rqn  t | t,  st-   j j |  } | d k rqn  | | _. qWn  |
 rx_ |
 j   D]N \ } } t | t/  s[t-   j j |  } | d k rq:n  | | _0 q:Wn  t1   }" xQ t2 |  d t3 j4 d  d t! D]. } | j d k r| j rd |" | j5 <qqWxH | D]@ } x7 | j D], } | j6 r	| j6 | _6 d |" | j5 <Pq	q	WqWx|  j7 D]p }# yG |# j8  d | d  d | d | pr	i  d | d | d | d | WqB	t9 k
 r	}$ t j: |$  qB	XqB	W j; |"  x6 | D]. } | j d  k r	 j< | j5 d | q	q	Wt    }%  j= d k	 r/
 j= d j |% |  n  d S(!   s   
        Add new computations to the internal dask graph

        This happens whenever the Client calls submit, map, get, or compute.
        i   R  R/  R  t   counti    c         3` s*   |  ]  } |   j  k o! |  k Vq d  S(   N(   R   (   Rf   t   dep(   RI   R   (    s4   lib/python2.7/site-packages/distributed/scheduler.pys	   <genexpr>  s    s+   User asked for computation on lost data, %ss   cancelled-keyR  Re   RJ   R  RC   RB   c         3` s   |  ] } |   k Vq d  S(   N(    (   Rf   Re  (   R  (    s4   lib/python2.7/site-packages/distributed/scheduler.pys	   <genexpr>.  s    R?   c         ` s   i  |  ] }   |  q S(    (    (   Rf   R   (   t   user_priority(    s4   lib/python2.7/site-packages/distributed/scheduler.pys
   <dictcomp>Y  s   	 g{Gz?Rw   t   reverseR@   R   t   restrictionsRu   R{   R^   s   update-graph-durationN(   RC   RB   (    (   RC   RB   (>   R$   R1   RH   Rj   R  Rn  R   R  R   R  R  R  R2  R   R|   R  R   t   coret   reverse_dictt   popR   Ru   R  R  Ro   R   Rq   Rt   t   extendR0  Rv   R   R   R  Rr   t   orderRw   R  R  Rx   Ry   R  R  R{   R   R   Rz   Ro  R   R   t   sortedR  R  Re   R   R%  R/  R   R   R  R1  R   (&   RI   RJ   R   R  Ru   R  Rw   R{   R^   t   submitting_taskR   R  RS   t   fifo_timeoutR  R   R  t   depsR   t   already_in_memoryRv   t   stackRe   Rg   R  t
   child_depsRe  t   touched_keyst   touched_taskst   dtsRr   R  t	   runnablesRz  R  R  R   t   end(    (   R  RI   R   R  s4   lib/python2.7/site-packages/distributed/scheduler.pyR/    sV   	'!$	+			#				*"	(		c         K` s   t  j d | |  |  j j |  } | d	 k r5 i  S|  j | } | j d k r |  j | d d | | } | j d k r | | j k s t	  q nU t  j d | | j | | j  | | j k r |  j
 | i d d 6| d 6 n  i  } | S(
   s@    Mark that a task has finished execution on a particular worker s   Stimulus task finished %s, %sRA   RC   Rb  sK   Received already computed task, worker: %s, state: %s, key: %s, who_has: %ss   release-taskR  Re   N(   R   t   debugR   R   Ro   R   R|   R  R   R   R  (   RI   Re   Rb  RX  Rg   Rh   R  (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyt   stimulus_task_finished  s&    
!c   	      K` s   t  j d | |  |  j j |  } | d k r5 i  S| j d k r | j } | d k r{ | d | _ |  j | d  } q |  j | d d | d | d	 | d
 | | } n i  } | S(   s3    Mark that a task has erred on a particular worker s   Stimulus task erred %s, %sRA   i    i   R@   RB   t   causeR   R   Rb  N(   R   R  R   R   Ro   R|   R   R  (	   RI   Re   Rb  R   R   RX  Rg   R   R  (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyt   stimulus_task_erred  s&    	c   
      K` s&  t    t j d | |  |  j j |  } | d k sJ | j d k rN i  S|  j j |  } t   } | d k	 r | j d k r x6 | j D]+ }	 |	 j	 j
 |  |	 j | j   8_ q W| j j   d | | <n  | r d | | <n  |  j |  |  j r| |  j k st  n  i  SWd QXd S(   s5    Mark that certain keys have gone missing.  Recover. s   Stimulus missing data %s, %sRC   R?   N(   R,   R   R  R   R   Ro   R|   R   R   RU   R  RZ   R   R  R  R   R   (
   RI   R  Re   Rb  t   ensureRX  Rg   t   ctsR  Rh   (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyt   stimulus_missing_data  s&    
		c         C` s:  t  j d | t |   | rF |  j | i d d 6t |  d 6 n  t |  } t   } g  } x{ | r | j   } | j |  g  |  j | j	 D] } | j
 d k r | j ^ q }	 |	 r | j |	  qd | j |  qd Wd   | D }
 |  j |
  |  j r0x( | D] } |  j | j st  qWn  t |  S(   Ns#   Client %s requests to retry %d keysR   R  R  RB   c         S` s   i  |  ] } d  |  q S(   R@   (    (   Rf   Re   (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pys
   <dictcomp>?  s   	 (   R   R  Rj   R  Rn  RH   R  R  R   Ru   R|   Re   R  R  R  R   R   R   Rm  (   RI   Rd  R  RJ   R  t   seent   rootsRe   R  t
   erred_depsR  (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyR9  ,  s&    '		5	c         ` s  t     j d k r d S   j k r0 d S j      t    }  j   }  j d   g i d d 6  d 6t | j  d 6 t j	 d	    | r t
 t t  '  j   j i d
 d 6t d 6 Wd QXn   j     j | d c | j 8< j | d j     j | j 8_  j | d sL j | =n   j j     j   = j | j = j j |   j j |   j   =d | _  j | j 8_ t   } x t | j  D] } | j }	 d | |	 <| s| j  d 7_  | j   j! k rp| |	 =t" j# t$ d |	 d | j%    d  }
  j& |	 d d |
 d |	 } | j' |  qpqqWxS | j( D]H } | j) j |  | j) s~| j* rd | | j <qd | | j <q~q~W| j( j+    j, |  xO  j- D]C } y | j. d  d    Wqt/ k
 r1}
 t j0 |
  qXqW j sOt j	 d  n  t1 j2    f d    } t3 t4 j5 j6 d   }  j7 j8 | |  t j9 d    Wd QXd S(   s   
        Remove worker from cluster

        We do this when a worker reports that it plans to leave or when it
        appears to be unresponsive.  This may send its tasks back to a released
        state.
        R  Ns   already-removedR  s   remove-workerR  Rb  s   processing-taskss   Remove worker %sR:  R  R  R  R  R?   i   R   t   last_workeriRB   R   R  R   R   s   Lost all workersc           ` s/      j  k r+    j k r+  j   =n  d  S(   N(   R   t   events(    (   RT   RI   (    s4   lib/python2.7/site-packages/distributed/scheduler.pyt   remove_worker_from_events  s    s*   distributed.scheduler.events-cleanup-delays   Removed worker %sR  (:   R,   R`   R   R  R   R  R   RA   R   R  R)   R  R"   R  R  R   t   remove_resourcesR   R[   R  R  R   R!  RY   R  t   discardR  R  R\   R   Rn  Re   R   R   t   picklet   dumpst   KilledWorkerRi   R  R  RU   R   Rt   R  R  R%  R7  R   R   R   t	   coroutineR1   R   R   R   R  t
   call_laterR  (   RI   Rd  RT   R  R:  Rd   Rh   R  Rg   R   R   t   rR  R  t   cleanup_delay(    (   RT   RI   s4   lib/python2.7/site-packages/distributed/scheduler.pyR7  H  s    
	+

			
!			c         C` sx   t  j d | t |   | rM |  j | i d d 6t |  d 6| d 6 n  x$ | D] } |  j | | d | qT Wd S(   s"    Stop execution on a list of keys s$   Client %s requests to cancel %d keysR   R  R  t   forceN(   R   R  Rj   R  t
   cancel_key(   RI   Rd  R  RJ   R  Re   (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyR8    s    (i   c   
      ` s;   j  j   }  j   } | d k s5 | j rp  rl  j j t j d       f d    n  d S| s | j | h k r x3 t	 | j
  D] }  j | j   d | q Wn  t j d  |   j i d d 6 d 6 | rt	 | j  n | g } x* | D]" }	  j d	  g d
 |	 j  qWd S(   s,    Cancel a particular key and all dependents g?c         ` s    j      d  S(   Ni   (   R  (   t   _(   RJ   Re   R   RI   (    s4   lib/python2.7/site-packages/distributed/scheduler.pyt   <lambda>  Rj  NR  s#   Scheduler cancels key %s.  Force=%ss   cancelled-keyR  Re   R  RJ   (   R   R   R   Ro   R   R  t
   add_futureR   R  Rn  Rv   R  Re   R   R  R  R2  RE   (
   RI   Re   RJ   R   R  Rg   R   R  R   R  (    (   RJ   Re   R   RI   s4   lib/python2.7/site-packages/distributed/scheduler.pyR    s    	( c         C` s   |  j  j |  } | d  k r8 t |  } |  j  | <n  x | D] } |  j j |  } | d  k r t | d   } |  j | <d | _ n  | j j |  | j	 j |  | j d k r? |  j
 | d | q? q? Wd  S(   NR?   RC   RB   RJ   (   RC   RB   (   R   R   Ro   RD   R   Rq   R|   R   R  RF   R1  (   RI   R  RJ   R   R   Rg   (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyR0    s    c   	      C` s  t  j d | |  |  j | } t   } x t |  D]r } |  j j |  } | d k	 r6 | | j k r6 | j j	 |  | j
 } | j	 |  | s | j |  q q6 q6 Wi  } xP | D]H } | j s d | | j <q | j d k r | j r d | | j <q q W|  j |  d S(   s&    Remove keys from client desired list s   Client %s releases keys: %sR   RB   R?   N(   R   R  R   RH   Rn  R   R   Ro   RF   R  R   R  Rv   Re   R|   R   R  (	   RI   R  RJ   R   t   tasks2Re   Rg   t   sR  (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyR2    s$    			c         C` s   t    |  j | _ d S(   s    Handle heartbeats from Client N(   R$   R   RG   (   RI   RJ   (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyR3    s    c         ` s   |  j  |     j d k s" t    j s2 t    j sB t    j sR t    j sb t  t   f d     j D  s t    |  j	 k s t  d  S(   NR?   c         3` s   |  ] }   | j  k Vq d  S(   N(   R   (   Rf   R  (   Rg   (    s4   lib/python2.7/site-packages/distributed/scheduler.pys	   <genexpr>  s    (
   R   R|   R   R   R~   R   R   R  Ru   R  (   RI   Re   (    (   Rg   s4   lib/python2.7/site-packages/distributed/scheduler.pyt   validate_released  s    &c         C` s   |  j  | } | j s t  | j s, t  | j s< t  | |  j k sQ t  xN | j D]C } t | j  | | j k d k s t  | | j k s[ t  q[ Wd  S(   Ni   (	   R   R~   R   R   R   R  Ru   R   R   (   RI   Re   Rg   R  (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyt   validate_waiting  s    (c         C` s   |  j  | } | j s t  | j } | s2 t  | | j k sG t  | j sW t  x5 | j D]* } | j sv t  | | j k sa t  qa Wd  S(   N(   R   R~   R   R   RA   R   Ru   R   (   RI   Re   Rg   Rh   R  (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyt   validate_processing  s    	c         C` s   |  j  | } | j s t  | j s, t  | j s< t  | |  j k sQ t  xJ | j D]? } | | j k | j d k k s t  | | j k s[ t  q[ Wd  S(   NR@   RA   (   R@   RA   (	   R   R   R   R   R~   R  Rv   R   R|   (   RI   Re   Rg   R  (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyt   validate_memory  s    $c         C` s   |  j  | } | |  j k s" t  | j s2 t  | |  j k sG t  | j sW t  | j sg t  x  | j D] } | j sq t  qq Wd  S(   N(   R   R  R   R~   R   R   Ru   (   RI   Re   Rg   R  (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyt   validate_no_worker$  s    c         C` s0   |  j  | } | j s t  | j s, t  d  S(   N(   R   R   R   R   (   RI   Re   Rg   (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyt   validate_erred.  s    c         C` s   y | d  k r$ |  j j |  } n  | d  k rC t j d |  nm | j   y& t |  d | j j d d   } Wn0 t	 k
 r t j
 d | j j d d   n X| |  WnB t k
 r } t j |  t r d d  l } | j   n    n Xd  S(   Ns   Key lost: %st	   validate_t   -R  s   self.validate_%s not foundi    (   Ro   R   R   R   R  R   t   getattrR|   t   replaceR  R  R   R   R   R   R   (   RI   Re   Rg   R\  R   R   (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyR.   3  s$    
&c         C` s  t  |  j |  j |  j  t |  j  t |  j  k sF t d   n  x |  j j   D] \ } } t | t	 t
 f  s t t |  | f   t | t  s t t |  | f   | j | k s t  | j sV | j s t  | |  j k s t  qV qV Wxi |  j j   D]X \ } } t | t  sCt t |  | f   | j | k sXt  |  j | |  qWx |  j j   D]{ \ } } | d  k st | t	  st t |  | f   t | t  st t |  | f   | j | k s|t  q|Wd   |  j j   D } d   |  j j   D }	 | |	 k sKt | |	 f   d }
 xX |  j j   D]G \ } } t t | j j    | j  d k  st  |
 | j 7}
 qaWt |
 |  j  d k  st |
 |  j f   d  S(   Ns'   Workers not the same in all collectionsc         S` s"   i  |  ] \ } } | j  |  q S(    (   RZ   (   Rf   Rz  Rh   (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pys
   <dictcomp>d  s   	 c         S` s2   i  |  ]( \ } } t  d    | j D  |  q S(   c         s` s   |  ] } | j    Vq d  S(   N(   R   (   Rf   Rg   (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pys	   <genexpr>f  s    (   R   RU   (   Rf   Rz  Rh   (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pys
   <dictcomp>f  s   	i    g:0yE>(   t   validate_stateR   R   R   RH   R  R  R   R   Rc  R   R   Rl   RR   RT   RA   R\   R  Rq   Re   R.   Ro   RD   RE   t   absR   R   R  (   RI   t   allow_overlapRz  Rh   R   Rg   R  R   t   at   bt   actual_total_occupancyRb  (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyR  K  s:    -'	'3'.c         C` sV  | d k	 rs y |  j | } | j |  Wqs t k
 r_ |  j d k rp t j d |  qp qs t k
 ro qs Xn  | d k r d | k r |  j j	 | d  } n  | d k r |  j j
   } n8 g  | j D]( } | j |  j k r |  j | j ^ q } xU | D]M } y | j |  Wqt k
 rM|  j d k rNt j d |  qNqXqWd S(   s   
        Publish updates to all listening Queues and Comms

        If the message contains a key then we only send the message to those
        comms that care about the key.
        Rc   s    Tried writing to closed comm: %sRe   N(   Ro   R  R  R"   R`   R   t   criticalR   R   R   R   R   RE   (   RI   R  Rg   RJ   Rd  t   commsR  (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyR  y  s,    (c         c` s  | d k	 s t  d | _ t j d |  |  j d | g i d d 6| d 6 t |  |  j | <z t d d d	 |  j	  } | j
 |  | |  j | <| j i d
 d 6 z" |  j d | d i | d 6 VWd |  j d |  t j d |  XWd | j   s|  j | j i d d 6 n  yN t   sl|  j | j   V|  j | =|  j d k rlt j d |  qln  Wn t k
 rn XXd S(   sY    Add client to network

        We listen to all future messages from this Comm.
        s   Scheduler->Clients   Receive client connection: %sR  s
   add-clientR  RJ   R  t   2msR  s   stream-startR  Rd  t   extraNs   Finished handling client %ss   stream-closedRc   s   Close client connection: %s(   Ro   R   RY   R   R  R  RD   R   R   R  R  R  R  t   handle_streamR4  R  R  R3   R:  R`   t	   TypeError(   RI   Rd  RJ   t   bcomm(    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyR5    s0    	$"	
c         ` s    j  d k r" t j d    n   j d   g i d d 6  d 6 y  j   } Wn t k
 rj n= X j d g  | j D] } | j ^ q~ d | j	   j   =t
 j    f d    } t t j j d	   }  j j | |  d
 S(   s    Remove client from network Rc   s   Remove client %sR  s   remove-clientR  RJ   R  c           ` s/      j  k r+    j k r+  j   =n  d  S(   N(   R   R  (    (   RJ   RI   (    s4   lib/python2.7/site-packages/distributed/scheduler.pyt   remove_client_from_events  s    s*   distributed.scheduler.events-cleanup-delayN(   R`   R   R  R  R   R   R2  RF   Re   RE   R   R  R1   R   R   R   R  R  (   RI   RJ   R   Rg   R#  R  (    (   RJ   RI   s4   lib/python2.7/site-packages/distributed/scheduler.pyR4    s    $	)
c   	      C` sd  y|  j  | } i d d 6| d 6| j d 6|  j |  d 6} | j rW | j | d <n  | j rm t | d <n  | j } | r d   | D | d	 <d
   | D | d <n  |  j r | r t | d	 j	    s t
  n  | j } t |  t k r | j |  n
 | | d <|  j | |  WnB t k
 r_} t j |  t rYd d l } | j   n    n Xd S(   s.    Send a single computational task to a worker s   compute-taskR  Re   Rw   t   durationRz   Rr   c         S` s5   i  |  ]+ } g  | j  D] } | j ^ q | j  q S(    (   R   RT   Re   (   Rf   R  Rh   (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pys
   <dictcomp>  s   	 R   c         S` s   i  |  ] } | j  | j  q S(    (   RZ   Re   (   Rf   R  (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pys
   <dictcomp>  s   	 RZ   R   i    N(   R   Rw   t   get_task_durationRz   Rr   R  Ru   R   R  R   R   Rt   Rl   R   R  R  R   R   R   R   R   R   (	   RI   Rb  Re   Rg   R  R  R   R   R   (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyt   send_task_to_worker  s8    
				
c         K` s   t  j t |   d  d  S(   Ni   (   R   R   R!   (   RI   R  (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyt   handle_uncaught_error	  s    c         K` sI   | |  j  k r d  St |  |  j d | d | |  } |  j |  d  S(   NRe   Rb  (   R   R.   R  R  (   RI   Re   Rb  R  R  (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyR)  	  s
    
c         K` s&   |  j  d | |  } |  j |  d  S(   NRe   (   R  R  (   RI   Re   R  R  (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyR*  	  s    c         K` sn   |  j  j |  } | d  k r" d  S|  j | } | j | k	 rB d  S|  j d | d t |  } |  j |  d  S(   NRe   R  (   R   R   Ro   R   R   R  R   R  (   RI   Re   Rb  RJ   R  Rg   Rh   R  (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyR+  	  s    c         K` s  t  j d | |  |  j j d | | f  |  j j |  } | d  k sT | j rX d  S| |  j k r |  j | } | | j k r | j j	 |  | j
 j	 |  | j | j   8_ q n  | j s | j r |  j i d | 6 q |  j i d | 6 n  d  S(   Ns$   handle missing data key=%s worker=%sR  R?   R   (   R   R  R'  R  R   R   Ro   R   R   R  RU   RZ   R   Rt   R  (   RI   Re   t   errant_workerRX  Rg   Rh   (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyR-  	  s    		c   
      ` s     j  | }   f d   | D } | | j @} | j | 8_ i  } xO | D]G } | j | j   8_ | j }	 |	 j |  |	 sL d | | j <qL qL W| r   j |  n  d  S(   Nc         ` s   h  |  ] }   j  |  q S(    (   R   (   Rf   R   (   RI   (    s4   lib/python2.7/site-packages/distributed/scheduler.pys	   <setcomp>1	  s   	 R?   (   R   RU   RZ   R   R   R  Re   R  (
   RI   R  R  Rb  Rh   R   t   removed_tasksR  Rg   t   wh(    (   RI   s4   lib/python2.7/site-packages/distributed/scheduler.pyR,  /	  s    	c   
      C` s   |  j  | } d |  j k r3 |  j d j |  n  | j } | d k rY t j d  d S| r | j } |  j j	 | d  } | } | s | }	 n d | d | }	 |	 |  j | <n  | j
 | j | 8_
 |  j | j | 8_ d | j | <|  j |  d S(   s    A task has seceded from the thread pool

        We stop the task from being stolen in the future, and change task
        duration accounting as if the task has stopped.
        t   stealings;   Received long-running signal from duplicate task. Ignoring.Ni    g      ?(   R   R$  t   remove_key_from_stealableR   Ro   R   R  Rs   R  R   R\   RA   R  R  (
   RI   Re   Rb  t   compute_durationRg   Rh   Rs   t   old_durationt   new_durationt   avg_duration(    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyR.  ?	  s(    			c         c` s   d | _  |  j | } | j |  t j d |  z" |  j d | d i | d 6 VWd | |  j k r | j   |  j d |  n  Xd S(   s   
        Listen to responses from a single worker

        This is the main loop for scheduler-worker interaction

        See Also
        --------
        Scheduler.handle_client: Equivalent coroutine for clients
        s   Scheduler connection to workers"   Starting worker compute stream, %sRd  R  Rb  NRT   (   RY   R  R  R   R  R   R  R7  (   RI   Rd  Rb  t   worker_comm(    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyR  `	  s    	"
c         ` s^   t    t  r!   |  |    n  | rJ t   f d   |  j D  rJ d S|  j j    d S(   sy   
        Add external plugin to scheduler

        See https://distributed.readthedocs.io/en/latest/plugins.html
        c         3` s$   |  ] } t  | t     Vq d  S(   N(   R   Rl   (   Rf   t   p(   R  (    s4   lib/python2.7/site-packages/distributed/scheduler.pys	   <genexpr>	  s    N(   R   Rl   R  R%  R  (   RI   R  t
   idempotentRX  (    (   R  s4   lib/python2.7/site-packages/distributed/scheduler.pyt
   add_pluginv	  s
    %c         C` s   |  j  j |  d S(   s'    Remove external plugin from scheduler N(   R%  R  (   RI   R  (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyt   remove_plugin	  s    c         C` sO   y |  j  | j |  Wn0 t t f k
 rJ |  j j |  j d | n Xd S(   s    Send message to worker

        This also handles connection failures by adding a callback to remove
        the worker on the next cycle.
        RT   N(   R  R  R"   R  R  R  R7  (   RI   Rb  R  (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyR  	  s    i   c         #` s  t    } x@   j sK t j d  Vt    | | k r t j d   q q W| d k rt d     j j   D } n8 g  | D] }	   j |	  ^ q{ }   f d   | D } t | t	  s t
  t | | d   j d t V\ }
 } }   j d | d | d	 |  | rL| t k r&t |  } n | }   j d
 |
 d | d |  Vn    j | d g i d d 6| d	 6t |  d 6 t j |
   d S(   sb    Send data out to workers

        See also
        --------
        Scheduler.broadcast:
        g?s   No workers foundc         S` s"   i  |  ] \ } } | j  |  q S(    (   R[   (   Rf   Rz  Rh   (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pys
   <dictcomp>	  s   	 c         ` s#   i  |  ] }   j  | j |  q S(    (   R   R[   (   Rf   Rz  (   RI   (    s4   lib/python2.7/site-packages/distributed/scheduler.pys
   <dictcomp>	  s   	 R   R  R   RZ   RJ   R  R   R  R  R   R  R  N(   R$   R   R   R  t   TimeoutErrorRo   R   R  R   R   R   R4   R   R   R   R  Rj   R   R  R   (   RI   Rd  t   dataR   RJ   R   t   timeoutR  R[   Rz  R  R   RZ   R  (    (   RI   s4   lib/python2.7/site-packages/distributed/scheduler.pyR   	  s*    	"% +c      
   c` sb  t  |  } i  } x\ | D]T } |  j j |  } | d k	 rc g  | j D] } | j ^ qG | | <q g  | | <q Wt | d |  j d t d | V\ } }	 }
 |	 s i d d 6| d 6} nrg  |	 D]+ } | |  j k r |  j | j	 n d ^ q } t
 j d |	 | |
  i d d 6|	 d	 6} t   x |
 D] } |  j d
 |  q,Wx |	 j   D] \ } } | skqSn  |  j | } t
 j d t |  t |   x | D] } |  j j |  } | d k	 r| | j k r| j j |  | j j |  | j | j   8_ |  j i d | 6 qqWqSWWd QX|  j d i d d 6t |  d 6 t j |   d S(   s    Collect data in from workers R   R:  t   serializersR  R`   R7  s-   Couldn't gather keys %s state: %s workers: %sR  R  RT   s'   Workers don't have promised key: %s, %sR?   NR  R   R  R  (   Rn  R   R   Ro   R   RT   R5   R   R   R|   R   R  R,   R7  R   R   Rc  R   RU   R  RZ   R   R  R  Rj   R   R   (   RI   Rd  R  R9  R   Re   Rg   Rh   R7  t   missing_keyst   missing_workerst   resultt   missing_statesRb  R   (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyR   	  sN    &(5
	%$c         C` s/   t  j d  x |  j D] } | j   q Wd  S(   Ns   Clear task state(   R   R  R"  R  (   RI   t
   collection(    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyR  	  s    i   c         #` s  t    t   j  } t j d  xI   j j   D]8 }   j d g  | j D] } | j	 ^ qO d | j
  q6 W  f d     j D } x` t   j  D]O } y    j d | d | | k  Wq t k
 r } t j d d t q Xq W  j   xF   j D]: }	 y |	 j    Wqt k
 r<} t j |  qXqWt j d	 |  g  | j   D]' }
 |
 d k	 r^t |
 d
   j ^ q^} z y t g  | D]( } | j d t d | d d t  ^ q } t j t d |  |  V} t d   | D  st j d | d t n  Wn! t j k
 r;t j d  n XWd x | D] } | j   qGWX  j      j! | d g i d d 6| d 6 t"   } x: t"   | d k  rt   j  | k  rt j# d  VqW  j$ i d d 6 Wd QXd S(   s*    Restart all workers.  Reset local state. s"   Send lost future signal to clientsR  RJ   c         ` s+   i  |  ]! }   j  | d  d t |  q S(   R  Rf  (   Ri  R  (   Rf   t   addr(   RI   (    s4   lib/python2.7/site-packages/distributed/scheduler.pys
   <dictcomp>
  s   	RT   R:  s+   Exception while restarting.  This is normalt   exc_infos   Send kill signal to nannies: %sR   R8  g?t   executor_waitt   secondsc         s` s   |  ] } | d  k Vq d S(   R  N(    (   Rf   t   resp(    (    s4   lib/python2.7/site-packages/distributed/scheduler.pys	   <genexpr>8
  s    s(   Not all workers responded positively: %ssV   Nannies didn't report back restarted within timeout.  Continuuing with restart processNR  R   R  i
   g{Gz?R  (%   R,   Rj   R   R   R  R   R   R2  RF   Re   RE   Rn  R7  R   R  R  R%  R   R   R  Ro   R   R   R(   R   R   t   with_timeoutR   R  R  R6  t	   close_rpcR  R  R$   R  R  (   RI   RJ   R8  t	   n_workersR   Rg   t   nanniesR?  R   R  t   nanny_addressR  t   respsR  (    (   RI   s4   lib/python2.7/site-packages/distributed/scheduler.pyR   
  sX    
	- 
' 5
$	+c         #` s&  | d k r3 | d k r* t  j  } q3 g  } n  | d k	 r} x; | D]0 } |  j k rF | j  j | d  qF qF Wn  | r g  | D] }  j | d d t ^ q }	 n | }	 t j     f d    }
 t	 g  |	 D] } | d k	 r |
 |  ^ q  V} t
 t t | |     d S(   s2    Broadcast message to workers, return all results R  R  Rf  c         3` sW   t  |  d  j d  j V} d | _ t | d t d    V} t j |   d  S(   NR   R   s   Scheduler BroadcastR:  R9  (   R   R   R   RY   R    R  R   R   (   R?  Rd  RC  (   R  RI   R9  (    s4   lib/python2.7/site-packages/distributed/scheduler.pyt   send_messagek
  s
    	N(   Ro   Rn  R   R   R  Ri  R  R   R  R(   R   R   t   zip(   RI   Rd  R  R   t   hostsR  R9  Rd   Rz  R  RJ  RT   t   results(    (   R  RI   R9  s4   lib/python2.7/site-packages/distributed/scheduler.pyR   N
  s    	".	/c      	   c` s?   |  j  d | d | d | g d |  V} t j | |   d S(   sB    Proxy a communication through the scheduler to some other worker Rd  R  R   R9  N(   R   R   R   (   RI   Rd  R  Rb  R9  Re  (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyR   z
  s    	c         #` s{  t    l| rq   f d   | D } g  | D] } | j s- | j ^ q- } | r t i d d 6| d 6  q n t   j j    }  r   f d    D   f d   | D } n% t   j j     d   | D } d    D } x< | j   D]. \ }	 }
 x |
 D] } | | j	 |	  qWq Wd	   | j   D } t
 | j    t |  } t t t t | j   d
 t d t   } t t |   } t |  } g  } x6| t   d  D] } d   | | D } t t | j   d
 t d t  } y x | | | k rx | | | k  r| | | k rt |  \ } } | | | k r| | j	 |  | j | | | f  | | c | 8<| | c | 7<qqW| | | k rt |  } qqWWqt k
 rPqXqWt d    } t t  } xL | D]D \ } } } | | j | j j | j  | | j j | j  qW  f d   | j   D V} x8 | j   D]* \ } }
   j | i d d 6|
 d 6 qW  j d i d d 6t |  d 6t t |  d 6t t |  d 6t |  d 6 t d   | j   D  sct i d d 6t
 g  | D] } d | k r1| d ^ q1g   d 6  n  x | D] \ } } } | j d k st  | j j	 |  | j j	 |  | j | j    7_   j! j d | j t"   | j | j f  qjW  f d   | j   D V} xL | D]D \ } } } | j j# |  | j j# |  | j | j    8_ qWt i d d 6  Wd QXd S(   s   Rebalance keys so that each worker stores roughly equal bytes

        **Policy**

        This orders the workers by what fraction of bytes of the existing keys
        they have.  It walks down this list from most-to-least.  At each worker
        it sends the largest results it can find and sends them to the least
        occupied worker until either the sender or the recipient are at the
        average expected load.
        c         ` s   h  |  ] }   j  |  q S(    (   R   (   Rf   R   (   RI   (    s4   lib/python2.7/site-packages/distributed/scheduler.pys	   <setcomp>
  s   	 s   missing-dataR`   R  c         ` s   h  |  ] }   j  |  q S(    (   R   (   Rf   Rz  (   RI   (    s4   lib/python2.7/site-packages/distributed/scheduler.pys	   <setcomp>
  s   	 c         ` s    i  |  ] } | j    @|  q S(    (   R   (   Rf   Rg   (   R   (    s4   lib/python2.7/site-packages/distributed/scheduler.pys
   <dictcomp>
  s   	 c         S` s   i  |  ] } | j  |  q S(    (   R   (   Rf   Rg   (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pys
   <dictcomp>
  s   	 c         S` s   i  |  ] } t    |  q S(    (   RH   (   Rf   Rh   (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pys
   <dictcomp>
  s   	 c         S` s/   i  |  ]% \ } } t  d    | D  |  q S(   c         s` s   |  ] } | j    Vq d  S(   N(   R   (   Rf   Rg   (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pys	   <genexpr>
  s    (   R   (   Rf   Rh   R   (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pys
   <dictcomp>
  s   	Re   R  i   c         S` s   i  |  ] } | j    |  q S(    (   R   (   Rf   Rg   (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pys
   <dictcomp>
  s   	 c           S` s
   t  t  S(   N(   R   Rn  (    (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyR  
  Rj  c         ` s7   i  |  ]- \ } }   j  d  |  j d |  |  q S(   R?  R   (   R   R   (   Rf   R  R   (   RI   (    s4   lib/python2.7/site-packages/distributed/scheduler.pys
   <dictcomp>
  s   	 R   R  R   R  s
   total-keyst   senderst
   recipientst
   moved_keysc         s` s   |  ] } | d  d k Vq d S(   R`   R  N(    (   Rf   R  (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pys	   <genexpr>
  s    RC   c         ` s=   i  |  ]3 \ } }   j  d  |  j d | d t  |  q S(   R?  R  R  (   R   t   delete_dataR   (   Rf   R  R   (   RI   (    s4   lib/python2.7/site-packages/distributed/scheduler.pys
   <dictcomp>
  s   	R  N($   R,   R   Re   R   RH   R   R   R   R   R  R   Rj   Rn  R   R   R  R   R  R   t   reversedt   nextR  t   StopIterationR   RT   R  R   R  R|   R   RU   RZ   R   R'  R$   R  (   RI   Rd  R  R   R   Rg   t   missing_datat   workers_by_taskt   tasks_by_workerR   R   t   vvR   t   avgt   sorted_workersRO  t	   recipientt   msgst   sendert   sender_keyst   nbt   to_recipientst
   to_sendersR<  R  (    (   RI   R   s4   lib/python2.7/site-packages/distributed/scheduler.pyR   
  s    
% *! 	"<	&c         #` s  | d k s t     f d     j |  D } | d k rL t |  } n t | t |   } | d k r| t d   n    f d   | D } g  | D] } | j s | j ^ q }	 |	 r t i d d 6|	 d 6  n  | rNt	 t
  }
 xh | D]` } | j | @} t |  | k r x8 t j | t |  |  D] } |
 | j |  q5Wq q Wg  |
 j   D]I \ } }   j d | j  j d g  | D] } | j ^ qd	 t  ^ qdVx |
 j   D] \ } } | j | 8_ x3 | D]+ } | j j |  | j | j   8_ qW  j | j i d
 d 6g  | D] } | j ^ q-d 6 qWn  x| rt	 t  } x t |  D] } | t | j | @ } | d k r| j |  qpn  t | | t | j   } | d k st   xN t j | | j |  D]3 } g  | j D] } | j ^ q| | j | j <qWqpW  f d   | j   D V} x | j   D]q \ } } | d d k r  j d | d t | |   n t j d |    j | i d d 6| | d 6 qYWqQW  j d i d d 6t |  d 6t |  d 6| d 6 d S(   s   Replicate data throughout cluster

        This performs a tree copy of the data throughout the network
        individually on each piece of data.

        Parameters
        ----------
        keys: Iterable
            list of keys to replicate
        n: int
            Number of replications we expect to see within the cluster
        branching_factor: int, optional
            The number of workers that can copy data in each generation.
            The larger the branching factor, the more data we copy in
            a single step, but the more a given worker risks being
            swamped by data requests.

        See also
        --------
        Scheduler.rebalance
        i    c         ` s   h  |  ] }   j  |  q S(    (   R   (   Rf   Rz  (   RI   (    s4   lib/python2.7/site-packages/distributed/scheduler.pys	   <setcomp>  s   	 s$   Can not use replicate to delete datac         ` s   h  |  ] }   j  |  q S(    (   R   (   Rf   R   (   RI   (    s4   lib/python2.7/site-packages/distributed/scheduler.pys	   <setcomp>&  s   	 s   missing-dataR`   R  R?  R  s   replicate-removeR  c         ` s7   i  |  ]- \ } }   j  d  |  j d |  |  q S(   R?  R   (   R   R   (   Rf   Rz  R   (   RI   (    s4   lib/python2.7/site-packages/distributed/scheduler.pys
   <dictcomp>V  s   	R  Rb  s+   Communication failed during replication: %ss   replicate-addR  R   R   s	   key-counts   branching-factorN(   R   t   workers_listRo   Rj   t   minR  R   Re   R   R   RH   t   randomt   sampleR  R   R   RT   RQ  R   RU   R  RZ   R   R  R   Rn  R   R   t   warning(   RI   Rd  R  R  R   t   branching_factort   deleteR   Rg   RU  t   del_worker_taskst   del_candidatesRh   t   gatherst	   n_missingR  t   wwsRM  Rz  R   (    (   RI   s4   lib/python2.7/site-packages/distributed/scheduler.pyR   
  sj    %#X2	 5#*c         ` s6  | d
 k r! | d
 k r! d } n  t   | rU t d   |  j j   D  rU g  S| d
 k rm d   } n  t | |  j j     d    j   D } d    j   D   t | j    } t   j    }    f d   } t  d | } g  }	 t	 |  j  }
 x | r| j
   } | d
 k rMt d    | D  rMPn  | rq|
 t	  |  | k  rqPn  | | | 8} | d
 k	 rt	 |	  | k  s| d
 k	 r| | | k r|	 j |  |
 t	  |  8}
 qPqWg  |	 D]  }  | D] } | j ^ qq} | r(t j d	 |  n  | SWd
 QXd
 S(   s	  
        Find workers that we can close with low cost

        This returns a list of workers that are good candidates to retire.
        These workers are not running anything and are storing
        relatively little data relative to their peers.  If all workers are
        idle then we still maintain enough workers to have enough RAM to store
        our data, with a comfortable buffer.

        This is for use with systems like ``distributed.deploy.adaptive``.

        Parameters
        ----------
        memory_factor: Number
            Amount of extra space we want to have for our stored data.
            Defaults two 2, or that we want to have twice as much memory as we
            currently have data.
        n: int
            Number of workers to close
        minimum: int
            Minimum number of workers to keep around
        key: Callable(WorkerState)
            An optional callable mapping a WorkerState object to a group
            affiliation.  Groups will be closed together.  This is useful when
            closing workers must be done collectively, such as by hostname.

        Examples
        --------
        >>> scheduler.workers_to_close()
        ['tcp://192.168.0.1:1234', 'tcp://192.168.0.2:1234']

        Group workers by hostname prior to closing

        >>> scheduler.workers_to_close(key=lambda ws: ws.host)
        ['tcp://192.168.0.1:1234', 'tcp://192.168.0.1:4567']

        Remove two workers

        >>> scheduler.workers_to_close(n=2)

        Keep enough workers to have twice as much memory as we we need.

        >>> scheduler.workers_to_close(memory_ratio=2)

        Returns
        -------
        to_close: list of worker addresses that are OK to close

        See Also
        --------
        Scheduler.retire_workers
        i   c         s` s   |  ] } | j  Vq d  S(   N(   RA   (   Rf   Rh   (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pys	   <genexpr>  s    c         S` s   |  j  S(   N(   RT   (   Rh   (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyR    Rj  c         S` s/   i  |  ]% \ } } t  d    | D  |  q S(   c         s` s   |  ] } | j  Vq d  S(   N(   RW   (   Rf   Rh   (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pys	   <genexpr>  s    (   R   (   Rf   R   R   (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pys
   <dictcomp>  s   	 c         S` s/   i  |  ]% \ } } t  d    | D  |  q S(   c         s` s   |  ] } | j  Vq d  S(   N(   RZ   (   Rf   Rh   (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pys	   <genexpr>  s    (   R   (   Rf   R   R   (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pys
   <dictcomp>  s   	 c         ` s0   t  d    |  D  }   |  } | | f S(   Nc         s` s   |  ] } | j  Vq d  S(   N(   RA   (   Rf   Rh   (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pys	   <genexpr>  s    (   R  (   t   groupt   is_idlet   bytes(   t   group_bytest   groups(    s4   lib/python2.7/site-packages/distributed/scheduler.pyRe     s    Re   c         s` s   |  ] } | j  Vq d  S(   N(   RA   (   Rf   Rh   (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pys	   <genexpr>  s    s   Suggest closing workers: %sN(   Ro   R,   R  R   R   R   R   R   R  Rj   R  R  R  RT   R   R  (   RI   t   memory_ratioR  Re   t   minimumt   limit_bytest   limitt   totalR  t   to_closet   n_remainRn  t   gRh   R<  (    (   Rq  Rr  s4   lib/python2.7/site-packages/distributed/scheduler.pyt   workers_to_closek  s@    5	
&	& -c         +` s@  t    1 d k r xk t r yJ   j |     rY   j d  d | d |  V n  t j    Wq t k
 r| q Xq Wn    f d    D  t   d k r t	 j
 g   D] } | j ^ q   }  f d   | D } n	 t	   } t	   j j     } | rf| rT  j d | d g  | D] }	 |	 j ^ q.d d	 d
 t  Vqft j g    n  d    D }
 | r|
 rg  |
 D] }   j d | d t  ^ qVn  | rx' |
 D] }   j d | d t  qWn    j d i d d 6|
 d 6t |  d 6   j t |
  i d d 6 t j |
   Wd QXd S(   s   Gracefully retire workers from cluster

        Parameters
        ----------
        workers: list (optional)
            List of worker IDs to retire.
            If not provided we call ``workers_to_close`` which finds a good set
        remove: bool (defaults to True)
            Whether or not to remove the worker metadata immediately or else
            wait for the worker to contact us
        close_workers: bool (defaults to False)
            Whether or not to actually close the worker explicitly from here.
            Otherwise we expect some external job scheduler to finish off the
            worker.
        **kwargs: dict
            Extra options to pass to workers_to_close to determine which
            workers we should drop

        Returns
        -------
        Dictionary mapping worker ID/address to dictionary of information about
        that worker for each retired worker.

        See Also
        --------
        Scheduler.workers_to_close
        R   R  R  c         ` s   h  |  ] }   j  |  q S(    (   R   (   Rf   Rz  (   RI   (    s4   lib/python2.7/site-packages/distributed/scheduler.pys	   <setcomp>  s   	 i    c         ` s+   h  |  ]! } | j  j    r | j  q S(    (   R   t   issubsetRe   (   Rf   Rg   (   R   (    s4   lib/python2.7/site-packages/distributed/scheduler.pys	   <setcomp>	  s   	 R  R  i   Rh  c         S` s"   i  |  ] } | j    | j  q S(    (   Rn   RT   (   Rf   Rh   (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pys
   <dictcomp>  s   	 Rb  R  RT   R  s   retire-workersR  s
   moved-keyst   retiredN(   R,   Ro   R  R{  R   R   R   R   Rj   RH   t   unionRU   R   R   R   RT   R   R  R7  R  Rn  (   RI   Rd  R   R  R  RX  Rz  R  t   other_workersRh   t   worker_keys(    (   RI   R   s4   lib/python2.7/site-packages/distributed/scheduler.pyR     sR    
		%		-c         C` s   | |  j  k r d S|  j  | } x | D] } |  j j |  } | d k	 r | j d k r | | j k r | j | j   7_ | j j |  | j	 j |  q q' |  j
 | i d d 6| g d 6t d 6 q' Wd S(	   s   
        Learn that a worker has certain keys

        This should not be used in practice and is mostly here for legacy
        reasons.  However, it is sent by workers from time to time.
        s	   not foundRC   s   delete-dataR  R  R  R  N(   R   R   R   Ro   R|   RU   RZ   R   R  R   R  R   (   RI   Rd  Rb  R  Rh   Re   Rg   (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyR   ,  s    &c      	   ` ss  t    d  f d   | j   D } t j d |  x| j   D] \ } }   j j |  } | d
 k r t | d
  }   j | <n  d | _ | | k r | j	 | |  n  xb | D]Z }	   j
 |	 }
 | |
 j k r |
 j | j   7_ |
 j j |  | j j |
  q q W  j i d d 6| d 6t |  d 6 qC W| ri  j d t |  d	 |  n  Wd
 QXd
 S(   s   
        Learn that new data has entered the network from an external source

        See Also
        --------
        Scheduler.mark_key_in_memory
        c         ` s;   i  |  ]1 \ } } g  | D] }   j  |  ^ q |  q S(    (   R  (   Rf   R   R   RX  (   RI   (    s4   lib/python2.7/site-packages/distributed/scheduler.pys
   <dictcomp>P  s   	 s   Update data %sRC   s   key-in-memoryR  Re   R   R  RJ   N(   R,   R   R   R  R   R   Ro   Rq   R|   R   R   RU   RZ   R   R  R   R  Rn  R0  (   RI   Rd  R   RZ   RJ   R9  Re   R   Rg   Rz  Rh   (    (   RI   s4   lib/python2.7/site-packages/distributed/scheduler.pyR   D  s(    

	&c         C` sY  | d  k | d  k d k s. t | | f   | d  k r y |  j | } Wq t k
 r |  j i d d 6| d 6d | d  SXn	 | j } | j d k r |  j i d d 6| d 6d | d | n | j d k r |  j i d	 d 6| d 6d | d | nV | j d
 k rU| j } |  j i d d 6| d 6| j d 6| j	 d 6d | d | n  d  S(   Ni   s   cancelled-keyR  Re   RJ   R   Rg   RC   s   key-in-memoryRB   s
   task-erredR   R   (
   Ro   R   R   R   R  Re   R|   R   R   R   (   RI   Re   Rg   RJ   t
   failing_ts(    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyR1  h  s,    .!	**	
t   1sc   
   
   k` s@  d d l  } t |  } t   | r: | j |  } n  | rR | j |  } n  | rj | j |  } n  | r| | |   n d } t | t j  r | V} n  z~ y` xY |  j d k r | d k r | |   }	 n | |  |  }	 | j	 |	  Vt j
 |  Vq WWn t t f k
 rn XWd | r5| |  |  n  XWd QXd S(   s   
        Provides a data Comm to external requester

        Caution: this runs arbitrary Python code on the scheduler.  This should
        eventually be phased out.  It is mostly used by diagnostics.
        i    NRc   (   R  R1   R,   t   loadsRo   R   R   t   FutureR`   R  R  t   EnvironmentErrorR"   (
   RI   Rd  t   functiont   setupR  R  RX  R  R|   R  (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyR     s0    


 c         ` sS   | d  k	 r8 t t   j |   }   f d   | D Sd     j j   D Sd  S(   Nc         ` s9   i  |  ]/ } g    j  | j D] } | j ^ q |  q S(    (   R   RA   Re   (   Rf   Rz  Rg   (   RI   (    s4   lib/python2.7/site-packages/distributed/scheduler.pys
   <dictcomp>  s   	 c         S` s8   i  |  ]. \ } } g  | j  D] } | j ^ q |  q S(    (   RA   Re   (   Rf   Rz  Rh   Rg   (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pys
   <dictcomp>  s   	 (   Ro   RH   R   R  R   R   (   RI   Rd  R   (    (   RI   s4   lib/python2.7/site-packages/distributed/scheduler.pyR>    s    c         ` s;   | d  k	 r    f d   | D Sd     j j   D Sd  S(   Nc         ` sN   i  |  ]D } |   j  k rA g    j  | j D] } | j ^ q, n g  |  q S(    (   R   R   RT   (   Rf   R   Rh   (   RI   (    s4   lib/python2.7/site-packages/distributed/scheduler.pys
   <dictcomp>  s   	c         S` s8   i  |  ]. \ } } g  | j  D] } | j ^ q |  q S(    (   R   RT   (   Rf   Re   Rg   Rh   (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pys
   <dictcomp>  s   	 (   Ro   R   R   (   RI   Rd  R  (    (   RI   s4   lib/python2.7/site-packages/distributed/scheduler.pyR=    s    c         ` sM   | d  k	 r2 t   j |  }   f d   | D Sd     j j   D Sd  S(   Nc         ` sN   i  |  ]D } |   j  k rA g    j  | j D] } | j ^ q, n g  |  q S(    (   R   RU   Re   (   Rf   Rz  Rg   (   RI   (    s4   lib/python2.7/site-packages/distributed/scheduler.pys
   <dictcomp>  s   	c         S` s8   i  |  ]. \ } } g  | j  D] } | j ^ q |  q S(    (   RU   Re   (   Rf   Rz  Rh   Rg   (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pys
   <dictcomp>  s   	 (   Ro   R   R  R   R   (   RI   Rd  R   (    (   RI   s4   lib/python2.7/site-packages/distributed/scheduler.pyR<    s
    c         ` sM   | d  k	 r2 t   j |  }   f d   | D Sd     j j   D Sd  S(   Nc         ` s2   i  |  ]( } |   j  k r   j  | j |  q S(    (   R   R[   (   Rf   Rz  (   RI   (    s4   lib/python2.7/site-packages/distributed/scheduler.pys
   <dictcomp>  s   	 c         S` s"   i  |  ] \ } } | j  |  q S(    (   R[   (   Rf   Rz  Rh   (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pys
   <dictcomp>  s   	 (   Ro   R   R  R   R   (   RI   Rd  R   (    (   RI   s4   lib/python2.7/site-packages/distributed/scheduler.pyR;    s    c   	      #` sI  | d  k	 r t |  } t   } xn | r | j   }   j | } | j d k ro | j d   | j D  q$ | j d k r$ | j |  q$ q$ Wt	 t  } xJ | D], } | j
 r | | j
 j j | j  q q Wn d     j D } | st j i    nB   f d   | j   D V} d   | j   D } t j |   d  S(   NR@   c         s` s   |  ] } | j  Vq d  S(   N(   Re   (   Rf   R  (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pys	   <genexpr>  s    RA   c         S` s   i  |  ] } d  |  q S(   N(   Ro   (   Rf   Rz  (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pys
   <dictcomp>  s   	 c         ` s4   i  |  ]* \ } }   j  |  j d  |  |  q S(   R  (   R   R   (   Rf   Rz  R   (   RI   (    s4   lib/python2.7/site-packages/distributed/scheduler.pys
   <dictcomp>  s   	 c         S` s%   i  |  ] \ } } | r | |  q S(    (    (   Rf   R   R   (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pys
   <dictcomp>  s   	 (   Ro   Rn  RH   R  R   R|   R  Ru   R  R   R   RT   R  Re   R   R   R   R   (	   RI   Rd  R  R  RA   Re   Rg   R   R  (    (   RI   s4   lib/python2.7/site-packages/distributed/scheduler.pyR?    s(    			$c      	   ` s   t     | d  k	 r/   f d   | D } n d     j j   D } | r t d    } x0 | j   D]" \ } } | t |  c | 7<qj Wt |  } n  | SWd  QXd  S(   Nc         ` s#   i  |  ] }   j  | j |  q S(    (   R   RZ   (   Rf   R   (   RI   (    s4   lib/python2.7/site-packages/distributed/scheduler.pys
   <dictcomp>  s   	 c         S` s1   i  |  ]' \ } } | j  d  k	 r | j  |  q S(   N(   RZ   Ro   (   Rf   R   Rg   (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pys
   <dictcomp>  s   		c           S` s   d S(   Ni    (    (    (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyR    Rj  (   R,   Ro   R   R   R   R-   R   (   RI   Rd  R  t   summaryR<  t   outR   R   (    (   RI   s4   lib/python2.7/site-packages/distributed/scheduler.pyR     s    
c         C` s"   t  d   | j | j D  t S(   so   
        Get the estimated communication cost (in s.) to compute the task
        on the given worker.
        c         s` s   |  ] } | j  Vq d  S(   N(   RZ   (   Rf   R  (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pys	   <genexpr>  s    (   R   Ru   RU   t	   BANDWIDTH(   RI   Rg   Rh   (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyt   get_comm_cost  s    g      ?c         C` sE   | j  } y |  j | SWn& t k
 r@ |  j | j |  | SXd S(   sv   
        Get the estimated computation cost of the given task
        (not including any communication cost).
        N(   Rs   R  R   R  R  (   RI   Rg   R   Rs   (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyR%  	  s    	c         C` sS   d d l  m } |  j d i d d 6| d 6 | |  | d | d | d | d	 | S(
   sp    Run a function within this process

        See Also
        --------
        Client.run_on_scheduler:
        i   (   t   runR  s   run-functionR  R  t   argsRX  t   wait(   Rb  R  R  (   RI   R  R  R  RX  R  R  (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyR     s    c         C` s   yp |  j  } xR | d  D]F } | | k sC t | | t t f  rS t   | | <n  | | } q W| | | d <Wn) t k
 r } d d  l } | j   n Xd  S(   Nii    (   R  R   R   Rn  R   R   R   (   RI   R  R  R   t   metadataRe   R   R   (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyR   !  s    	&c         C` sc   |  j  } x | d  D] } | | } q Wy | | d SWn$ t k
 r^ | t k rX | S  n Xd  S(   Ni(   R  R   R/   (   RI   R  R  R   R  Re   (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyR   .  s    	c         ` s     f d   | D S(   Nc         ` s8   i  |  ]. } |   j  k r+   j  | j n d  |  q S(   N(   R   R|   Ro   (   Rf   Re   (   RI   (    s4   lib/python2.7/site-packages/distributed/scheduler.pys
   <dictcomp><  s   	 (    (   RI   R  R  (    (   RI   s4   lib/python2.7/site-packages/distributed/scheduler.pyR   :  s    c         C` sn   d d l  m } |  j | d t g  |  j D] } t | |  r- | ^ q- d } | j d | d | d |  S(   Ni    (   t   TaskStreamPluginR3  R  Rw  R  (   t#   distributed.diagnostics.task_streamR  R4  R  R%  R   t   collect(   RI   Rd  R  Rw  R  R  R2  Rg   (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyR   ?  s    /c         c` sc   | d k r t j i    n  |  j j |  |  j d t d d d |   V} t j |   d S(   s9    Registers a setup function, and call it on every worker R  R  R  R  N(   Ro   R   R   R(  R  R   R   (   RI   Rd  R  t	   responses(    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyR   F  s
    "c         C` s   | j  } d | _  | j } | |  j k r | j j |  } | j sc |  j | j 8_ d | _ n |  j | 8_ | j | 8_ |  j |  |  j	 | |  | r |  j
 | |  q n  d S(   s?   
        Remove *ts* from the set of processing tasks.
        i    N(   R   Ro   RT   R   RA   R  R  R\   R  t   release_resourcesR  (   RI   Rg   t   send_worker_msgRh   Rz  R$  (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyt   _remove_from_processingU  s    				c         K` s  |  j  r! | | j k s! t  n  | j j |  | j j |  | j | j   7_ | j } t |  d k r t	 | d t
 j d  d t } n  xI | D]A } | j }	 | |	 k r |	 j |  |	 s d | | j <q q q WxH | j D]= } | j }	 |	 j |  |	 r | j r d | | j <q q W| j rP| j rPd | | j <n= i d d 6| j d 6}
 | d k	 r| |
 d	 <n  |  j |
  d
 | _ | | _ |  j d } | | j k r|  j d d d | j g  n  d S(   s9   
        Add *ts* to the set of in-memory tasks.
        i   Re   Rw   R  RA   R?   s   key-in-memoryR  Rl   RC   s   fire-and-forgetRJ   R  N(   R   RU   R   R   R  RZ   R   Rv   Rj   R  R  R  R  R~   R  Re   Ru   R   R   Ro   R  R|   Rl   R   RF   R2  (   RI   Rg   Rh   R  Rl   R  RX  R  R  R  R  R   (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyt   _add_to_memoryi  s<    		$				c         C` s  y|  j  | } |  j r{ | j s( t  | j s8 t  | j sH t  | j sX t  t d   | j D  s{ t  n  | j	 r i d | 6Sd | _
 t   } x4 | j D]) } | j r | j | _ d | | <| Sq Wxb | j D]W } | j } | j s| j j |  n  | j
 d k r)d | | <q | j j |  q Wd   | j D | _ | j s|  j rrd | | <q|  j j |  d | _
 n  | SWnB t k
 r} t j |  t rd	 d  l } | j   n    n Xd  S(
   Nc         s` s   |  ] } | j  d  k Vq d S(   R   N(   R|   (   Rf   R  (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pys	   <genexpr>  s    R   R@   RB   R?   c         S` s%   h  |  ] } | j  d  k r |  q S(   R@   (   R|   (   Rf   R  (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pys	   <setcomp>  s   	 RA   s	   no-workeri    (   R   R   Rt   R   R~   R   R   R  Ru   R}   R|   R   R   Re   R  R   Rv   R   R  R   R   R   R   R   R   (   RI   Re   Rg   R  R  R  R   R   (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyRD    sJ    	#				
				c         C` s  y<|  j  | } |  j ra | |  j k s. t  | j s> t  | j sN t  | j sa t  n  |  j j |  | j r i d | 6St	   } xb | j
 D]W } | j } | j s | j j |  n  | j d k r d | | <q | j j |  q Wd | _ | j s7|  j rd | | <q7|  j j |  d | _ n  | SWnB t k
 r} t j |  t rzd d  l } | j   n    n Xd  S(   NR   R?   R@   RA   s	   no-workeri    (   R   R   R  R   R~   R   R   R  R}   R   Ru   Re   R  R|   R   R   R   R   R   R   R   R   (   RI   Re   Rg   R  R  R  R   R   (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyRL    s>    								c         C` s  |  j  |  } | rF | j rF |  j rF |  j j |  d | _ d S| j s[ | t k	 r t	 | |  j j
   | t |  j |   } n |  j r t |  j  d k  r t |  j d t j d  } qF|  j |  j t |  j  } n_ t |  j  d k  r#t |  j j
   d t j d  } n# |  j j
   |  j t |  j  } |  j r| d k st | t  st t |  | f   | j |  j k st  n  | S(   sJ   
        Decide on a worker for task *ts*.  Return a WorkerState.
        s	   no-workeri   Re   R\   N(   R  R{   R   R  R  R|   Ro   Ru   R  t   decide_workerR   R   t   worker_objectiveR  Rj   Rc  R  R  R  R   R   RR   R   Rl   RT   (   RI   Rg   R  Rb  (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyR    s2    		! $#		c   	      C` s  y|  j  | } |  j r | j s) t  | j s9 t  | j sI t  | j sY t  | j si t  | |  j k s~ t  t	 d   | j
 D  s t  n  |  j |  } | d  k r i  S| j } |  j |  } |  j | |  } | | | j | <| | _ | j | | 7_ |  j | | 7_ d | _ |  j | |  |  j |  |  j d 7_ | j rz| j j |  n  |  j | |  i  SWnB t k
 r} t j |  t rd d  l } | j    n    n Xd  S(   Nc         s` s   |  ] } | j  Vq d  S(   N(   R   (   Rf   R  (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pys	   <genexpr>'  s    RA   i   i    (!   R   R   R~   R   R   R   R   R}   R  R  Ru   R  Ro   RT   R%  R  RA   R\   R  R|   t   consume_resourcesR  R  Rr   RS   R  R&  R   R   R   R   R   R   (	   RI   Re   Rg   Rh   Rb  R$  Rd  R   R   (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyRF    sD    	"				c   
      K` s;  y |  j  | } |  j | } |  j r] | j s6 t  | j sE t  | j d k s] t  n  | j j   | d  k	 r | j	 |  n  |  j
 |  t   } |  j | | | |  |  j r | j s t  | j s t  | j s t  n  | SWnB t k
 r6} t j |  t r0d d  l }	 |	 j   n    n Xd  S(   NR@   i    (   R   R   R   R   R   R~   R|   R  Ro   R   R  R   R  R   R   R   R   R   R   R   (
   RI   Re   RZ   Rb  RX  Rh   Rg   R  R   R   (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyRG  J  s0    			c         K` sU  y|  j  | } | s t  t | t t f  s7 t  |  j r | j sO t  | j }	 | |	 j k sm t  | j s} t  | j	 s t | | j	 f   | j
 s t  | j d k s t  n  |  j j |  }	 |	 d  k r i d | 6S|	 | j k	 rt j d | j |	 |  i  S| ryg  | D]' \ }
 } } |
 d k r&| | f ^ q&} | rl| d \ } } qd  } } n
 d  } } | r|	 j j | t  r| j } |  j j | d  } | | } | s| } n d | d | } | |  j | <x |  j j | d	  D]y } | j r
| j } | j | } |  j | |  } | | | j | <| j | | | 7_ |  j | | | 7_ q
q
Wn  | d  k	 r| j |  n  t   } |  j |  |  j | |	 | d | d | |  j r| j st  | j st  n  | SWnB t k
 rP} t j |  t rJd d  l  } | j!   n    n Xd  S(
   NRA   R?   s^   Unexpected worker completed task, likely due to work stealing.  Expected: %s, Got: %s, Key: %st   computei    g      ?Rl   R  (    ("   R   R   R   Rc  R   R   R   RA   R~   R   R   R|   R   R   Ro   R   R  R  Rs   R  R  R  R  R\   R  R   R   R  R  R   R   R   R   R   (   RI   Re   RZ   Rl   R  Rb  t
   startstopsRX  Rg   Rh   R  R  R  t   Lt   compute_startt   compute_stopRs   R.  R/  R0  t   ttsRm  t   oldRd  R  R   R   (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyRI  m  sv    
		4
	
			!		c   	      C` sV  y|  j  | } |  j rU | j s) t  | j s9 t  | rU | j sR t  qU n  | j r x! | j D] } | j j	 |  qh W| j
 r | | _ d | _ i d | j 6Sn  t   } xR | j D]G } | j d k r d | | j <q | j d k r | j j |  q q Wxa | j D]V } | j j |  | j | j   8_ |  j | j i d d 6| g d 6t d	 6 qW| j j   d
 | _ |  j i d d 6| d 6 | j sd | | <n5 | j rd | | <n | j
 s| j rd | | <n  |  j r| j st  n  | SWnB t k
 rQ} t j |  t rKd d  l } | j    n    n Xd  S(   Ns   Worker holding Actor was lostRB   s	   no-workerRA   R@   s   delete-dataR  R  R  R?   s	   lost-dataRe   R   i    (   s	   no-workerRA   (!   R   R   R~   R   R   R   Rr   R   RS   R  R   R   R   Re   R   R|   R  RU   R  RZ   R   R  RT   R   R  R  Rt   R}   R   R   R   R   R   (	   RI   Re   R  Rg   Rh   R  R  R   R   (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyRP    sV    						)				c         C` s<  y |  j  | } |  j rq t d t  D | j s8 t  | j sH t  | j sX t  | j sh t  Wd  QXn  i  } | j } x3 | j	 D]( } | | _ | j s d | | j
 <q q W|  j i d d 6| d 6| j d 6| j d 6 d | _ | SWnB t k
 r7} t j |  t r1d d  l } | j   n    n Xd  S(	   NR   RB   s
   task-erredR  Re   R   R   i    (   R   R   R,   R   R   R   R   R~   R   Rv   Re   R  R   R   R|   R   R   R   R   (   RI   Re   Rg   R  R  R  R   R   (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyRQ    s8    				
	c         C` sY  y|  j  | } |  j r t d t  c t d   | j D  sH t  | j sW t  | j sg t  | j	 sw t  | j
 s t  Wd  QXn  t   } d  | _ d  | _ d  | _ x0 | j D]% } | j d k r d | | j <q q W|  j i d d 6| d 6 d | _ | SWnB t k
 rT} t j |  t rNd	 d  l } | j   n    n Xd  S(
   NR   c         s` s   |  ] } | j  d  k Vq d S(   RB   N(   R|   (   Rf   R  (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pys	   <genexpr>6  s    RB   R@   s   task-retriedR  Re   R?   i    (   R   R   R,   R   R  Ru   R   R   R   R~   R   R   Ro   R   R   Rv   R|   Re   R  R   R   R   R   (   RI   Re   Rg   R  R  R   R   (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyRO  0  s2    						c         C` sL  y|  j  | } |  j r< | j s) t  | j s< t  n  i  } xW | j D]L } | j } | | k rL | j |  | r | j r d | | j	 <q qL qL W| j
 j   d | _ | j r d | | <n6 | j r | j s | j r d | | <n | j j   | SWnB t k
 rG} t j |  t rAd d  l } | j   n    n Xd  S(   NR?   R   R@   i    (   R   R   R   R   R   Ru   R   R  R   Re   R~   R  R|   R}   R   R   R   R   R   R   R   (   RI   Re   Rg   R  R  R  R   R   (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyRE  R  s4    				c         C` s  yr|  j  | } |  j rg | j s( t  | j s8 t  | j sH t  |  j  | j d k sg t  n  |  j | d i d d 6| d 6d | _ t   } | j	 r d | | <n | j
 s | j r d | | <n  | j |  d k rQxZ | j D]O } | j d k r | j
 } | j |  | r=| j r=d | | j <q=q q W| j
 j   n  |  j rm| j smt  n  | SWnB t k
 r} t j |  t rd	 d  l } | j   n    n Xd  S(
   NRA   R  s   release-taskR  Re   R?   R   R@   i    (   R   R   R   R   R   R~   R|   R  R   R}   R   R   R   Ru   R  Re   R  R   R   R   R   R   R   (   RI   Re   Rg   R  R  R  R   R   (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyRH  v  s@    						c         K` sR  y	|  j  | } |  j r` | s. | j s. t  | j s= t  | j sM t  | j s` t  n  | j r | j } | j j	 |  n  |  j
 |  | d  k	 r | | _ n  | d  k	 r | | _ n  | d  k	 r |  j  | } | | _ n	 | j } i  }	 x' | j D] }
 | |
 _ d |	 |
 j <q WxH | j D]= }
 |
 j } | j |  | r*|
 j r*d |	 |
 j <q*q*W| j j   d | _ |  j i d d 6| d 6| j d 6| j d 6 |  j d } | | j k r|  j d	 d d
 | g  n  |  j r| j st  n  |	 SWnB t k
 rM} t j |  t rGd d  l } | j   n    n Xd  S(   NRB   R?   s
   task-erredR  Re   R   R   s   fire-and-forgetRJ   R  i    (   R   R   R   R   R   R   R~   Rr   RS   R  R  Ro   R   R   Rv   Re   Ru   R   R  R   R  R|   R  R   RF   R2  R   R   R   R   R   (   RI   Re   R  R   R   RX  Rg   Rh   R  R  R  R  R   R   R   (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyRJ    s`    							
	c         C` s   y |  j  | } |  j rX |  j  | j d k s5 t  | j sE t  | j sX t  n  |  j j |  d | _ x! | j D] } | j	 j
 |  q{ W| j	 j   i  SWnB t k
 r } t j |  t r d d  l } | j   n    n Xd  S(   Ns	   no-workerR?   i    (   R   R   R|   R   R   R~   R  R  Ru   R   R  R  R   R   R   R   R   R   (   RI   Re   Rg   R  R   R   (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyRK    s$    		c         C` s   |  j  j |  } | j d k s' t  |  j j |  x! | j D] } | j j |  qA W| j j	   d  | _ d  | _ | _ | _ | |  j k r |  j | =n  d  S(   NR   (   R   R  R|   R   R  R  R   RF   R  R  Ro   R   R   R   R   R  (   RI   Re   Rg   R   (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyt
   remove_key  s    	c         C` s  d | _  | j } xY | j D]N } t | _ | j j |  | j j |  | j  d k r d | | j <q q W| j j	   | j
 j	   xm | j D]b } | j j |  | j
 } | j |  | j r | j r | | k	 s t  d | | j <q q W| j j	   | j j	   xy | j D]n } | j j |  | j | j   8_ | j } | |  j k r|  j | i d d 6| g d 6t d 6 qqW| j j	   d  S(	   NR   RC   RB   s   delete-dataR  R  R  (   RC   RB   (   R|   Re   Rv   R  R}   Ru   R  R~   R  R  R   R   R   R   RU   RZ   R   RT   R   R  R   (   RI   Rg   R  Re   R  R  Rh   Rz  (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyt   _propagate_forgotten  s6    					)c         C` sP  y|  j  | } |  j r | j d k s. t  | j s> t  | j sN t  | j sZ q | j rf q | j r | j	 r | j
 r q d s t | f   n  i  } | j r x$ | j D] } | j j |  q Wn  |  j | |  |  j d |  |  j |  | SWnB t k
 rK} t j |  t rEd d  l } | j   n    n Xd  S(   NRC   i    Rg   (   R   R   R|   R   R   R~   Rt   R}   R   R   Rv   Rr   R   RS   R  R  R1  R  R   R   R   R   R   R   (   RI   Re   Rg   R  Rh   R   R   (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyRN  0  s6    				c         C` s?  y |  j  | } |  j r | j d k s. t  | j s> t  | j sN t  | j sm t | | j f   | j sy q | j r q | j	 r | j
 r | j r q d s t | f   n  i  } |  j | |  |  j d |  |  j |  | SWnB t k
 r:} t j |  t r4d d  l } | j   n    n Xd  S(   NR?   RB   i    Rg   (   R?   RB   (   R   R   R|   R   R   R   R~   Rt   R}   R   R   Rv   R  R1  R  R   R   R   R   R   R   (   RI   Re   Rg   R  R   R   (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyRM  X  s2    			c         O` s  yy |  j  | } Wn t k
 r( i  SX| j } | | k rB i  S|  j rl t | j  } t | j  } n  | | f |  j k r |  j | | f }	 |	 | | |  }
 n d | | f k rN|  j d | f }	 | r | s t  |  j	 | d  } | | k r|  j d | | f }	 n  |	 |  } | j
   } | j |  | }
 d } n t d | | f   | j } |  j j | | | |
 t   f  |  j rt j d | | | | j t |
   n  |  j r| j d k ry | | _ | | _ Wn t k
 rn X| |  j  | j <n  x[ t |  j  D]J } y | j	 | | | | |  Wq%t k
 rnt j d d t q%Xq%W| j d k r|  j  | j =qn  |
 SWnK t k
 r} t j d | | |  t rd d	 l } | j   n    n Xd	 S(
   st   Transition a key from its current state to the finish state

        Examples
        --------
        >>> self.transition('x', 'waiting')
        {'x': 'processing'}

        Returns
        -------
        Dictionary of recommendations for future transitions

        See Also
        --------
        Scheduler.transitions: transitive version of this function
        R?   s#   Impossible transition from %r to %rs5   Transitioned %r %s->%s (actual: %s).  Consequence: %sR   s   Plugin failed with exceptionR@  s$   Error transitioning %r from %r to %ri    N(   R   R   R|   R%  RH   Rv   Ru   RR  R   R  t   copyR  t   RuntimeErrorR&  R  R$   R   R   R  R   Re   Rn  R   R  R  R   R   R   R   (   RI   Re   t   finishR  RX  Rg   R  Rv   Ru   R\  R  R  R  t   finish2R  R   R   (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyR  |  sv    				"			c         C` s   t    } | j   } xH | r_ | j   \ } } | j |  |  j | |  } | j |  q W|  j r x | D] } |  j |  qp Wn  d S(   s    Process transitions until none are left

        This includes feedback from previous transitions and continues until we
        reach a steady state
        N(   RH   R  t   popitemR  R  R  R   R.   (   RI   R  R  Re   R  t   new(    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyR    s    			c         G` sI   t  |  } g  |  j D]/ } | d | k s? | j | d  r | ^ q S(   s6    Get all transitions that touch one of the input keys i    i   (   RH   R&  t   intersection(   RI   R  t   t(    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyt   story  s    c         C` sT   |  j  | } | j d k r  d S| r< | j j | k r< d S|  j i d | 6 d S(   s|    Reschedule a task

        Things may have shifted and this task may now be better suited to run
        elsewhere
        RA   NR?   (   R   R|   R   RT   R  (   RI   Re   Rb  Rg   (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyR     s    c         C` s  |  j  d k s | j d k r" d  S| d  k r: | j } n  | j } t | j  } |  j |  j  } | | k  s | | | d k  r |  j j	 |  |  j
 j |  nq |  j j |  | | | | | } | | k r| d k r| d | k r|  j
 j	 |  n |  j
 j |  d  S(   Ni    R  i   g?gffffff?(   R  R`   Ro   R\   R[   Rj   RA   R  R  R  R  R  (   RI   Rh   t   occt   ncR2  RY  t   pending(    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyR    s    	 (c         ` sS  t  } | j r+   f d   | j D } n  | j r g  | j D] }   j |  ^ q> } g  | D]& } |   j k r`   j | d ^ q` } | r t j |   n t   } | t  k r | } q | | O} n  | j r+  f d   | j j   D } t j	 | j
     } | t  k r| } q+| | M} n  | t  k r;| S  f d   | D Sd S(   s   Return set of currently valid workers for key

        If all workers are valid then this returns ``True``.
        This checks tracks the following state:

        *  worker_restrictions
        *  host_restrictions
        *  resource_restrictions
        c         ` s%   h  |  ] } |   j  k r |  q S(    (   R   (   Rf   Rz  (   RI   (    s4   lib/python2.7/site-packages/distributed/scheduler.pys	   <setcomp>   s   	 R  c         ` s<   i  |  ]2 \ }     f d     j  | j   D |  q S(   c         ` s(   h  |  ] \ } } |   k r |  q S(    (    (   Rf   Rz  t   supplied(   t   required(    s4   lib/python2.7/site-packages/distributed/scheduler.pys	   <setcomp>1  s   		(   R^   R   (   Rf   t   resource(   RI   (   R  s4   lib/python2.7/site-packages/distributed/scheduler.pys
   <dictcomp>0  s   	c         ` s   h  |  ] }   j  |  q S(    (   R   (   Rf   Rz  (   RI   (    s4   lib/python2.7/site-packages/distributed/scheduler.pys	   <setcomp>B  s   	 N(   R  Ry   Rx   t   coerce_hostnameR   RH   R~  Rz   R   R  R   (   RI   Rg   R  t   ht   hrt   ssRz  t   ww(    (   RI   s4   lib/python2.7/site-packages/distributed/scheduler.pyR    s(    
		%3			c         C` sC   | j  r? x3 | j  j   D] \ } } | j | c | 7<q Wn  d  S(   N(   Rz   R   Rb   (   RI   Rg   Rh   R  R  (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyR  D  s    	c         C` sC   | j  r? x3 | j  j   D] \ } } | j | c | 8<q Wn  d  S(   N(   Rz   R   Rb   (   RI   Rg   Rh   R  R  (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyR  I  s    	c         C` sq   |  j  | } | r& | j j |  n  i  | _ x; | j j   D]* \ } } d | j | <| |  j | | <q? Wd S(   Ni    R  (   R   R^   R  Rb   R   (   RI   R  Rb  R^   Rh   R  t   quantity(    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyRC  R  s    	c         C` s?   |  j  | } x+ | j j   D] \ } } |  j | | =q Wd  S(   N(   R   R^   R   (   RI   Rb  Rh   R  R  (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyR  \  s    c         C` s   | |  j  k r |  j  | } n  t | t  r= t |   } n  t | t j  se t d | f   n  | rz t |  } n t |  } | S(   s   
        Coerce possible input addresses to canonical form.
        *resolve* can be disabled for testing with fake hostnames.

        Handles strings, tuples, or aliases.
        s-   addresses should be strings or tuples, got %r(	   R!  R   Rm  R   t   sixt   string_typesR!  R   R   (   RI   R?  t   resolve(    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyR  a  s    c         C` s,   | |  j  k r$ |  j |  j  | j S| Sd S(   s2   
        Coerce the hostname of a worker.
        N(   R!  R   Rd   (   RI   Rd   (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyR  w  s    c         ` sy   | d k r t |  j  St   } xJ | D]B   d   k rK | j    q) | j   f d   |  j D  q) Wt |  S(   s   
        List of qualifying workers

        Takes a list of worker addresses or hostnames.
        Returns a list of all worker addresses that match
        Rk  c         ` s"   h  |  ] }   | k r |  q S(    (    (   Rf   R  (   Rz  (    s4   lib/python2.7/site-packages/distributed/scheduler.pys	   <setcomp>  s   	 N(   Ro   Rn  R   RH   R  R  (   RI   R   R  (    (   Rz  s4   lib/python2.7/site-packages/distributed/scheduler.pyRb    s    	$c         C` sW   d d l  m } |  j d k rJ | d |  j d i |  d 6d t  |  _ n  |  j j   S(   sU   Start an IPython kernel

        Returns Jupyter connection info dictionary.
        i   (   R   R  t   nsR   R'  N(   t   _ipython_utilsR   R  Ro   R  R   t   get_connection_info(   RI   Rd  R   (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyR     s
    %c         C` s   t  g  | j D]! } | | j k r | j   ^ q  } | j | j } | t | } | j rw t | j	  | | j
 f S| | j
 f Sd S(   s   
        Objective function to determine which worker should get the task

        Minimize expected start time.  If a tie then break with data storage.
        N(   R   Ru   R   R   R\   R[   R  Rr   Rj   RS   RZ   (   RI   Rg   Rh   R  t
   comm_bytest
   stack_timet
   start_time(    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyR    s    4	c         #` s   | d  k r  j } n t  j  t |  @}      f d   | D V} | ro t j | j     } n  t j |   d  S(   Nc      	   ` s:   i  |  ]0 }  j  |  j d   d  d    |  q S(   R  Rw  Re   (   R   R#   (   Rf   Rz  (   Re   RI   R  Rw  (    s4   lib/python2.7/site-packages/distributed/scheduler.pys
   <dictcomp>  s   	 (   Ro   R   RH   R#   R
   R   R   R   (   RI   Rd  R   t   merge_workersR  Rw  Re   R<  (    (   Re   RI   R  Rw  s4   lib/python2.7/site-packages/distributed/scheduler.pyR@    s    
 c         #` sZ  | p t  j j d    t   d d   | d  k rB  j } n t  j  t |  @}    f d   | D V} g  | j   D] } | d ^ q }	 t j	 t
 |	     f d    }	 g  |	 D]' \ }
 } |
 t t d |   f ^ q }	 t   } xM | j   D]? } x6 | d D]* \ } } x | D] } | j |  q*WqWqWd	   | D } g  | j   D] } | d ^ qf} t t
 d
 t |   } d } x | D] \ } } |     } | | k r | } x0 | j   D] \ } } | j | d g  qWn  x2 | j   D]$ \ } } | | d d c | 7<qWqWt j i |	 d 6| d 6  d  S(   Ns    distributed.worker.profile.cycleR   R   c         ` s4   i  |  ]* }   j  |  j d   d   |  q S(   R  Rw  (   R   t   profile_metadata(   Rf   Rz  (   RI   R  Rw  (    s4   lib/python2.7/site-packages/distributed/scheduler.pys
   <dictcomp>  s   	 t   countsc         ` s   |  d     S(   Ni    (    (   R  (   t   dt(    s4   lib/python2.7/site-packages/distributed/scheduler.pyR    Rj  i   R  c         S` s   i  |  ] } g  |  q S(    (    (   Rf   R   (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pys
   <dictcomp>  s   	 Re   i    i(   R   R   R   R1   Ro   R   RH   R   t	   itertoolsR   R   R   R   R  Rn  R   R   R  R   R   (   RI   Rd  R   R  R  Rw  t   profile_cycle_intervalR<  R   R  R$   Rn  R  R  Re  R   t   groups1t   groups2t   lastt   tt(    (   R  RI   R  Rw  s4   lib/python2.7/site-packages/distributed/scheduler.pyt   get_profile_metadata  s8    
	#!4	# c         C` s   |  j  } | d  k r' t | j  } n< | j } g  t t | t |    D] } | | ^ qL } g  | D] } | j | j |  f ^ qj S(   N(	   R  Ro   Rn  R   R  Rc  Rj   t	   levelnamet   format(   RI   Rd  R  t   deque_handlerR  R  R  (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyRA    s    		3c         c` s:   |  j  d i d d 6| d 6d |  V} t j |   d  S(   NR  RA  R  R  R   (   R   R   R   (   RI   Rd  R  R   RM  (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyRB    s    'c   	      C` sK  d } y|  j  d k r d St   } t d |  } |  j j   d k  r t |  j j    } x t t	 |   D] } | | t	 |  } | d 7} z- | d k s | j r wq n  |  j |  Wd ~ Xt   | } | d k rq t d | d  } Pqq qq Wn  |  j j | |  j d	 | Wn' t k
 rFt j d
 d t   n Xd S(   s   Periodically reassess task duration time

        The expected duration of a task can change over time.  Unfortunately we
        don't have a good constant-time way to propagate the effects of these
        changes out to the summaries that they affect, like the total expected
        runtime of each of the workers, or what tasks are stealable.

        In this coroutine we walk through all of the workers and re-align their
        estimates with the current state of tasks.  We do this periodically
        rather than at every transition, and we only do it if the scheduler
        process isn't under load (using psutil.Process.cpu_percent()).  This
        lets us avoid this fringe optimization when we have better things to
        think about.
        g?R  NRB  i2   i   g{Gzt?i   t   worker_indexs   Error in reevaluate occupancyR@  (   R`   R$   R   R   t   cpu_percentRn  R   R   R  Rj   Ro   RA   t   _reevaluate_occupancy_workerR  t   add_timeoutR  R   R   R  R  (	   RI   R  t   DELAYR  t	   next_timeR   R  Rh   R$  (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyR    s2    	
	c   	      C` s   | j  } d } d } xQ | j D]F } |  j |  } |  j | |  } | | | j | <| | | 7} q W| | _  |  j | | 7_ |  j |  | | d k r d |  j k r |  j d } x. | j D]  } | j |  | j |  q Wn  d S(   s    See reevaluate_occupancy i    g?R+  N(	   R\   RA   R%  R  R  R  R$  R,  t   put_key_in_stealable(	   RI   Rh   R  R  RZ   Rg   R$  Rd  t   steal(    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyR  8  s     		c         C` si   t    } xY |  j j   D]H } | j | |  j k  r t j d |  j |  |  j d | j  q q Wd  S(   Ns9   Worker failed to heartbeat within %s seconds. Closing: %sRT   (	   R$   R   R   RG   R   R   Rf  R7  RT   (   RI   R  Rh   (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyRT  O  s    	c         C` s   t  d   |  j j   D  r# d  S|  j r0 d  S|  j sU t   |  j |  j k } n' |  j d d } t   | |  j k } | r |  j j	 |  j
  n  d  S(   Nc         s` s   |  ] } | j  Vq d  S(   N(   RA   (   Rf   Rh   (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pys	   <genexpr>[  s    i(   R  R   R   R  R&  R$   R   R   R  R  R:  (   RI   R:  t	   last_task(    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyRV  Z  s    		(   RN   RO   RP   t   default_portRo   t   ALLOWED_FAILURESR   RK   RL   Rn   Ri  Rv  Rx  R  R  R   R  R  R:  R  R   R   R6  R/  R  R  R  R9  R7  R8  R  R0  R2  R3  R  R  R  R  R  R  R.   R  R  R5  R4  R&  R'  R)  R*  R+  R-  R,  R.  R  R4  R5  R  R   R   R  R   R   R   R   R   R{  R   R   R   R1  R   R>  R=  R<  R;  R?  R   R  R%  R   R   R/   R   R   R   R   R  R  RD  RL  R  RF  RG  RI  RP  RQ  RO  RE  RH  RJ  RK  R  R  RN  RM  R  R  R  t   transition_storyR   R  R  R  R  RC  R  R  R  Rb  R   R  R@  R  RA  RB  R  R  RT  RV  (    (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyR     s  F 		$	>;	'hc
	
			
	
	.%!	)	
!		(3	K$zflS##		+	3	,	(	.&Y:	)	"	$	-D			"	(	$	U			1			
					)	0		c         C` s  |  j  } t d   | D  s% t  |  j r7 | } n/ t g  | D] } | j D] } | ^ qN qA  } | t k r | s | } q nH | t |  @} | s | } | s |  j r t	 |  | t |  Sd Sq n  | s d St |  d k r t |  St | d | S(   sK  
    Decide which worker should take task *ts*.

    We choose the worker that has the data on which *ts* depends.

    If several workers have dependencies then we choose the less-busy worker.

    Optionally provide *valid_workers* of where jobs are allowed to occur
    (if all workers are allowed to take the task, pass True instead).

    If the task requires data communication because no eligible worker has
    all the dependencies already, then we choose to minimize the number
    of bytes sent between workers.  This is determined by calling the
    *objective* function.
    c         s` s   |  ] } | j  Vq d  S(   N(   R   (   Rf   R  (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pys	   <genexpr>{  s    i   Re   N(   Ru   R  R   Rr   R	   R   R  RH   R{   R  Ro   Rj   R   Rc  (   Rg   t   all_workersR  t	   objectiveR  t
   candidatesR  Rh   (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyR  j  s(    			/	

c         ` s    j  t k s*   j  d k s* t       j rr   j j   j  sr t d t   j  t   j  f   n    j r   j j   j  s t d t   j  t   j  f   n  xl   j D]a } | j	 s t d t    t |  f   | j  d k s t d t    t |  f   q Wx   j D] }   | j k sut d t    t |  t | j  f     j  d k r|   j k s| j	 st d
 t    t |  f   n  | j  d k s3t  q3WxA   j D]6 } | j  d k st d t    t |  f   qWxb   j D]W }   | j k sit d t    t |  t | j  f   | j  d k s't  q'W  j
 d k	   j  d	 k k st  t   j	    j  d k k st     j	 f     j  d	 k r8t d     j D  s%t d t    t   j  f     j s8t  n    j	 r#  j sw  j swt d t    t   j	  f     j r  j st  t   j t  st  n  t   f d     j D  st  xP   j	 D]B }   | j k st d t    t |  t | j  f   qWn    j rxP   j D]B }   | j k s6t d t    t |  t | j  f   q6Wn    j r  j  d k rt g    j	 D] }   | j k ^ q d k st  n    j  d	 k r    j
 j k st  qn  d S(   s'   
    Validate the given TaskState.
    R   s"   waiting not subset of dependenciess    waiters not subset of dependentss   waiting on in-memory depR?   s   waiting on released deps   not in dependency's dependentsR@   RA   s   dep missings   waiter not in plays   not in dependent's dependenciesRC   c         s` s   |  ] } | j  Vq d  S(   N(   R   (   Rf   R  (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pys	   <genexpr>  s    s    task processing without all depss   unneeded task in memoryc         3` s   |  ] }   | j  k Vq d  S(   N(   R~   (   Rf   R  (   Rg   (    s4   lib/python2.7/site-packages/distributed/scheduler.pys	   <genexpr>  s    s   not in who_has' has_whats   not in who_wants' wants_whati   N(   R@   RA   (   R@   RA   (   R|   t   ALL_TASK_STATESR   R~   R|  Ru   Rc  R   Rv   R   R   Ro   R   R  R   Rt   Rl   R   R  RU   RF   Rr   R   RS   (   Rg   R  Rh   R   (    (   Rg   s4   lib/python2.7/site-packages/distributed/scheduler.pyR     s    *		+4						$3				&						:c         C` s}   xM |  j  D]B } |  | j k s
 t d t |   t |  t | j  f   q
 Wx& |  j D] } | j d k sZ t  qZ Wd  S(   Ns   not in has_what' who_hasRC   RA   (   RC   RA   (   RU   R   R   Rc  RS   R|   (   Rh   Rg   (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyt   validate_worker_state  s    		c         C` s   x |  j    D] } t |  q Wx | j    D] } t |  q. Wxd | j    D]V } xM | j D]B } | | j k s_ t d t |  t |  t | j  f   q_ WqO Wd S(   s   
    Validate a current runtime state

    This performs a sequence of checks on the entire graph, running in about
    linear time.  This raises assert errors if anything doesn't check out.
    s   not in wants_what' who_wantsN(   R   R   R  RF   R   R   Rc  (   R   R   R   Rg   Rh   R   (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyR    s    		s   rechunk-splits   shuffle-splitc         C` s8   |  d k r d S|  d k  r  d S|  d k  r0 d Sd Sd S(	   sR   
    Interval in seconds that we desire heartbeats based on number of workers
    i
   g      ?i2   i   i   i   i   N(    (   R  (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyR     s    R   c           B` s   e  Z d    Z RS(   c         C` s/   t  t |   j | |  | |  _ | |  _ d  S(   N(   RS  R   RK   R   R  (   RI   R   R  (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyRK   /  s    	(   RN   RO   RK   (    (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyR   .  s   (   t
   __future__R    R   R   t   collectionsR   R   R   t   datetimeR   t	   functoolsR   R  R  R  t   numbersR   R  R|  R  Rd  R  Rq  R   R  t   cytoolzR	   R
   R   R   R   t   ImportErrort   toolzR   R   R   R   t   tornadoR   t   tornado.genR   t   tornado.ioloopR   R   t   batchedR   Rd  R   R   R   R   t   compatibilityR   R   R   R   R  R   R   R    R!   R"   Rj  R#   RX   R$   t   nodeR%   t	   proctitleR&   R  R'   t   utilsR(   R)   R*   R+   R,   R-   R.   R/   R0   R1   R2   R3   t
   utils_commR4   R5   t
   utils_perfR6   R7   t   publishR8   t   queuesR9   t   recreate_exceptionsR:   t   lockR;   t   pubsubR<   R+  R=   t   variableR>   t	   getLoggerRN   R   R   R   R  R  R   R   RW  R  R  t   objectRD   RR   Rq   R   R   R   R   R   R   R   R   R   R  R   R  R  t   _round_robinR  R  R   R   (    (    (    s4   lib/python2.7/site-packages/distributed/scheduler.pyt   <module>   s   ,,"""(R	# L					               	+	a				