
\c           @` s  d  d l  m Z m Z m Z d  d l Z d  d l m Z m Z d  d l m	 Z	 d  d l
 Z
 d  d l Z d  d l Z d  d l m Z d  d l Z d  d l Z d  d l Z d  d l Z d  d l Z d  d l Z d  d l m Z d  d l m Z y  d  d l m Z m Z m Z Wn- e k
 r9d  d l m Z m Z m Z n Xd  d	 l m  Z  d  d
 l! m" Z" d  d l# m$ Z$ d  d l% m& Z& d d l' m( Z( m) Z) d d l* m+ Z+ d d l) m, Z, m- Z- m. Z. d d l/ m0 Z0 d d l1 m2 Z2 m3 Z3 m4 Z4 m5 Z5 d d l6 m7 Z7 m8 Z8 m9 Z9 m: Z: m; Z; d d l< m= Z= d d l> m? Z? d d l@ mA ZA d d lB mC ZC d d lD mE ZE d d lF m Z mG ZG mH ZH mI ZI d d lJ mK ZK d d lL mM ZM d d lN mO ZN d d lP mQ ZQ mR ZS d d lT mU ZU mV ZV mW ZW mX ZX mY ZY mZ ZZ m[ Z[ m\ Z\ m] Z] m^ Z^ m_ Z_ m` Z` ma Za mb Zb mc Zc md Zd me Ze mf Zf mg Zg mh Zh d d li mj Zj mk Zk d d  ll mm Zm mn Zn mo Zo e\ jp   Zq e jr es  Zt e ju jv d!  Zw d" Zx y d  d ly Zy ey jz   j{ Z| Wn* e k
 r et j} d#  d$ Z| e~ Zy n Xd% d& d' d( f Z d% d& d) f Z d% d& d) d' d( f Z d& d) f Z eK g Z g  Z d* eA f d+     YZ d,   Z e~ d- e d.  Z d/   ZR d0 e f d1     YZ eq d2  Z e" j e~ e~ e~ e~ d3   Z d  g Z e~ e~ e~ e~ d4  Z d5   Z y/ d  d6 l m Z e d7 e   d8 d9   Z Wn e k
 r>e   Z n Xd:   Z d;   Z e g Z e j d< d=  Z d>   Z d?   Z d@   Z e~ dA  Z e~ dB  Z dC   Z e" j f  i  e~ e dD   Z d S(E   i    (   t   print_functiont   divisiont   absolute_importN(   t   defaultdictt   deque(   t	   timedelta(   t   PicklingError(   t   istask(   t   apply(   t   pluckt   partialt   merge(   t   Return(   t   gen(   t   IOLoop(   t   Eventi   (   t   profilet   comm(   t   BatchedSend(   t   get_address_hostt   get_local_address_fort   connect(   t   offload(   t   unicodet   get_thread_identityt   finalizet   MutableMapping(   t   error_messaget   CommClosedErrort	   send_recvt   pingpongt   coerce_to_address(   t	   WorkSpace(   t   time(   t
   ServerNode(   t   preload_modules(   t   setproctitle(   t   picklet   to_serializet   deserialize_bytest   serialize_bytelist(   t   PubSubWorkerExtension(   t   Security(   t   safe_sizeof(   t   ThreadPoolExecutort   secede(   t   funcnamet   typenamet   get_ipt   has_argt   _maybe_complext
   log_errorst   ignoringt
   mp_contextt   import_filet   silence_loggingt   thread_statet   json_load_robustt	   key_splitt   format_bytest   DequeHandlert   PeriodicCallbackt   parse_bytest   parse_timedeltat   iscoroutinefunctiont   warn_on_duration(   t	   pack_datat   gather_from_workers(   t   ThrottledGCt   enable_gc_diagnosist   disable_gc_diagnosiss   distributed.admin.pdb-on-errs   --no-value-sentinel--s3   Please install psutil to estimate worker memory useg    eAt   waitingt   readyt	   executings   long-runningt   constrainedt   Workerc           B` s  e  Z d  Z e e e e e d e e e e d e e e e e e e e d e e e e j j d  d  Z d   Z	 d   Z
 e d    Z d	   Z e d
  Z e j d    Z e j d    Z e j d    Z d   Z e j e e e d   Z e d  Z e j e e d   Z e e d  Z d   Z e j d d   Z d   Z d d  Z d   Z e j e d e e d   Z d   Z d   Z  e j e d   Z! e j d    Z" d   Z# e j e e e e d    Z$ e e e e d!  Z% e j e e e d"   Z& e j d#    Z' e e e e e e e e e e( d$ 
 Z) d%   Z* e d&  Z+ e e d'  Z, e d(  Z- e d)  Z. d*   Z/ d+   Z0 e d,  Z1 d-   Z2 e d.  Z3 d/   Z4 e5 e d0  Z6 e d1  Z7 e d2  Z8 d3   Z9 d4   Z: d5   Z; d6   Z< e d7  Z= d8   Z> e j e d9   Z? d:   Z@ e j d;    ZA e j d<    ZB d=   ZC d>   ZD e e e d?  ZE e( d@  ZF dA   ZG e j f  e e dB   ZH f  e e dC  ZI f  e e dD  ZJ e j e e e f  i  dE   ZK e e e dF  ZL dG   ZM dH   ZN e j e( dI   ZO e j dJ    ZP dK   ZQ dL   ZR e e e e dM  ZS e d e dN  ZT e e dO  ZU dP   ZV dQ   ZW dR   ZX dS   ZY dT   ZZ dU   Z[ dV   Z\ dW   Z] dX   Z^ dY   Z_ e dZ    Z` d[ d\  Za d]   Zb RS(^   s   Worker node in a Dask distributed cluster

    Workers perform two functions:

    1.  **Serve data** from a local dictionary
    2.  **Perform computation** on that data and on data from peers

    Workers keep the scheduler informed of their data and use that scheduler to
    gather data from other workers when necessary to perform a computation.

    You can start a worker with the ``dask-worker`` command line application::

        $ dask-worker scheduler-ip:port

    Use the ``--help`` flag to see more options

        $ dask-worker --help

    The rest of this docstring is about the internal state the the worker uses
    to manage and track internal computations.

    **State**

    **Informational State**

    These attributes don't change significantly during execution.

    * **ncores:** ``int``:
        Number of cores used by this worker process
    * **executor:** ``concurrent.futures.ThreadPoolExecutor``:
        Executor used to perform computation
    * **local_dir:** ``path``:
        Path on local machine to store temporary files
    * **scheduler:** ``rpc``:
        Location of scheduler.  See ``.ip/.port`` attributes.
    * **name:** ``string``:
        Alias
    * **services:** ``{str: Server}``:
        Auxiliary web servers running on this worker
    * **service_ports:** ``{str: port}``:
    * **total_out_connections**: ``int``
        The maximum number of concurrent outgoing requests for data
    * **total_in_connections**: ``int``
        The maximum number of concurrent incoming requests for data
    * **total_comm_nbytes**: ``int``
    * **batched_stream**: ``BatchedSend``
        A batched stream along which we communicate to the scheduler
    * **log**: ``[(message)]``
        A structured and queryable log.  See ``Worker.story``

    **Volatile State**

    This attributes track the progress of tasks that this worker is trying to
    complete.  In the descriptions below a ``key`` is the name of a task that
    we want to compute and ``dep`` is the name of a piece of dependent data
    that we want to collect from others.

    * **data:** ``{key: object}``:
        Dictionary mapping keys to actual values
    * **task_state**: ``{key: string}``:
        The state of all tasks that the scheduler has asked us to compute.
        Valid states include waiting, constrained, executing, memory, erred
    * **tasks**: ``{key: dict}``
        The function, args, kwargs of a task.  We run this when appropriate
    * **dependencies**: ``{key: {deps}}``
        The data needed by this key to run
    * **dependents**: ``{dep: {keys}}``
        The keys that use this dependency
    * **data_needed**: deque(keys)
        The keys whose data we still lack, arranged in a deque
    * **waiting_for_data**: ``{kep: {deps}}``
        A dynamic verion of dependencies.  All dependencies that we still don't
        have for a particular key.
    * **ready**: [keys]
        Keys that are ready to run.  Stored in a LIFO stack
    * **constrained**: [keys]
        Keys for which we have the data to run, but are waiting on abstract
        resources like GPUs.  Stored in a FIFO deque
    * **executing**: {keys}
        Keys that are currently executing
    * **executed_count**: int
        A number of tasks that this worker has run in its lifetime
    * **long_running**: {keys}
        A set of keys of tasks that are running and have started their own
        long-running clients.

    * **dep_state**: ``{dep: string}``:
        The state of all dependencies required by our tasks
        Valid states include waiting, flight, and memory
    * **who_has**: ``{dep: {worker}}``
        Workers that we believe have this data
    * **has_what**: ``{worker: {deps}}``
        The data that we care about that we think a worker has
    * **pending_data_per_worker**: ``{worker: [dep]}``
        The data on each worker that we still want, prioritized as a deque
    * **in_flight_tasks**: ``{task: worker}``
        All dependencies that are coming to us in current peer-to-peer
        connections and the workers from which they are coming.
    * **in_flight_workers**: ``{worker: {task}}``
        The workers from which we are currently gathering data and the
        dependencies we expect from those connections
    * **comm_bytes**: ``int``
        The total number of bytes in flight
    * **suspicious_deps**: ``{dep: int}``
        The number of times a dependency has not been where we expected it

    * **nbytes**: ``{key: int}``
        The size of a particular piece of data
    * **types**: ``{key: type}``
        The type of a particular piece of data
    * **threads**: ``{key: int}``
        The ID of the thread on which the task ran
    * **active_threads**: ``{int: key}``
        The keys currently running on active threads
    * **exceptions**: ``{key: exception}``
        The exception caused by running a task if it erred
    * **tracebacks**: ``{key: traceback}``
        The exception caused by running a task if it erred
    * **startstops**: ``{key: [(str, float, float)]}``
        Log of transfer, load, and compute times for a task

    * **priorities**: ``{key: tuple}``
        The priority of a key given by the scheduler.  Determines run order.
    * **durations**: ``{key: float}``
        Expected duration of a task
    * **resource_restrictions**: ``{key: {str: number}}``
        Abstract resources required to run a task

    Parameters
    ----------
    scheduler_ip: str
    scheduler_port: int
    ip: str, optional
    data: MutableMapping, type, None
        The object to use for storage, builds a disk-backed LRU dict by default
    ncores: int, optional
    loop: tornado.ioloop.IOLoop
    local_dir: str, optional
        Directory where we place local resources
    name: str, optional
    memory_limit: int, float, string
        Number of bytes of memory that this worker should use.
        Set to zero for no limit.  Set to 'auto' to calculate
        as TOTAL_MEMORY * min(1, ncores / total_cores)
        Use strings or numbers like 5GB or 5e9
    memory_target_fraction: float
        Fraction of memory to try to stay beneath
    memory_spill_fraction: float
        Fraction of memory at which we start spilling to disk
    memory_pause_fraction: float
        Fraction of memory at which we stop running new tasks
    executor: concurrent.futures.Executor
    resources: dict
        Resources that this worker has like ``{'GPU': 2}``

    Examples
    --------

    Use the command line to start a worker::

        $ dask-scheduler
        Start scheduler at 127.0.0.1:8786

        $ dask-worker 127.0.0.1:8786
        Start worker at:               127.0.0.1:1234
        Registered with scheduler at:  127.0.0.1:8786

    See Also
    --------
    distributed.scheduler.Scheduler
    distributed.nanny.Nanny
    s   dask-worker-spacet   autot   200mss$   distributed.worker.profile.low-levelc   '      K` s  t    |  _ t    |  _ t    |  _ t    |  _ t    |  _ t    |  _ t    |  _ t t	  |  _
 t t  |  _ t j   |  _ t   |  _ t    |  _ t    |  _ t j j d  |  _ t j j d  |  _ d |  _ d |  _ t d    |  _ t	   |  _ t    |  _ t    |  _ t    |  _ t    |  _ t    |  _  t j   |  _! t    |  _" t t# j$  |  _% t d d  |  _& t# j$   |  _' t d d  |  _( t    |  _) d |  _* t    |  _+ t t,  |  _- t    |  _. t,   |  _/ t   |  _0 t	   |  _1 d |  _2 t	   |  _3 d  |  _5 t d t j j d   |  _6 d	 |  _7 t d d
  |  _8 | j9 d t:  |  _; i |  j< dV 6|  j= dW 6|  j= dX 6|  j> dY 6|  j? dZ 6|  j@ d[ 6|  jA d\ 6|  jA d] 6|  jA d^ 6|  jB d_ 6|  jA d` 6|  jA da 6|  jA db 6|  _C i |  jD dc 6|  jE dd 6|  jF de 6|  jG df 6|  _H t d d
  |  _I d |  _J t d d
  |  _K d |  _L d |  _M d |  _N d  |  _O | j9 d t j j d   } tP | d d } |  jQ   | rtR |  } | d } nc | d  k rt j j d d   rt j j d  } n- | d  k rtS |  } n tS | | f  } d |  _T | pBtU |  _V | pQi  |  _W | p`i  jX   |  _Y | |  _Z | |  _[ |  j[ d  k rt j j d  |  _[ n  | |  _\ |  j\ d  k rt j j d  |  _\ n  | |  _] tP | d d |  _^ t    |  __ | rt` d |  n  ta d d  G tb tc jd je |   |  _f |  jf jg d  d!  |  _h |  jh ji |  _j Wd  QX| pytk   |  _l tm |  jl tk  stn  |  jl jo d"  |  _p |  jl jq d"  |  _r ts | |  jV  |  _t t: |  _u d# | k r | j9 d#  |  _v n t j j d$  |  _v d% | k r6| j9 d%  |  _w n t j j d&  |  _w d' | k rl| j9 d'  |  _x n t j j d(  |  _x tm | ty  r| |  _z nt{ |  r|   |  _z ntm | t|  r| d | d)   |  _z n |  jt r|  jv s|  jw ry  d d* l} m~ } m } m } Wn t k
 r:t d+   n Xtc jd j |  jj d,  }  | t t d- d. t | |    }! t t |  jt  |  jv  }" | i  |! |" t  |  _z n t    |  _z i  |  _ | pt j   |  _ d  |  _ t   |  _ |
 |  _ | pt |  jV d/ d0 |  _ t d) d/ d1 |  _ |	 |  _ d |  _ t    |  _ t: |  _ d  |  _ |  jj t jd k rt jd j d |  jj  n  i  |  _ | pi  |  _ | pi  |  _ | rt  |  n i  |  _ | |  _ i |  j d2 6|  j d3 6|  j d4 6|  j d5 6|  j d6 6|  j d7 6|  j d8 6t d9 6|  j d: 6|  j d; 6|  j d< 6|  j d= 6|  j d> 6|  j d? 6|  j d@ 6|  j dA 6|  j dB 6|  j dC 6}# i |  j dD 6|  j dE 6t |  j dF t: dG 6|  j dH 6|  j dI 6}$ t t |   j dJ |# dK |$ dL |  j dM |  jp |  |  j |  |  _ i |  j j dN 6|  j dO 6|  d" 6|  _ t |  j dP dL |  j }% |% |  j dQ <| |  _ |  jt r
t: |  _ t |  j |  j^ dP dL |  j }% |% |  j d <n  | d  k r
t } n  x | D] }& |& |   q
Wt dR t  |  _ t dS  t |  j tP t j j dT  d d dP dL |  j }% |% |  j d= <t |  j | dP dL |  j }% |% |  j dU <t j t j |    d  S(g   Ns'   distributed.worker.connections.outgoings'   distributed.worker.connections.incomingg    cAi    c           S` s   d S(   Ni    (    (    (    (    s1   lib/python2.7/site-packages/distributed/worker.pyt   <lambda>D  t    t   maxleni  s+   distributed.comm.recent-messages-log-lengthg    ׇAi t   validateRG   RH   t   memoryt   errorRI   RJ   t   rescheduleds   long-runningt   flightt   profile_cycle_intervals    distributed.worker.profile.cyclet   defaultt   mst   addresss   scheduler-addresss   distributed.worker.preloads   distributed.worker.preload-argvt   levelt   1ss   Creating scratch directories is taking a surprisingly long time. This is often due to running workers on a network file system. Consider specifying a local-directory to point workers to write scratch data to a local disk.t   prefixs   worker-t   workert   memory_target_fractions    distributed.worker.memory.targett   memory_spill_fractions   distributed.worker.memory.spillt   memory_pause_fractions   distributed.worker.memory.pausei   (   t   Buffert   Filet   Funcs3   Please `pip install zict` for spill-to-disk workerst   storaget   on_errort   raiset   thread_name_prefixs   Dask-Worker-Threads's   Dask-Actor-Threadst   gathert   runt   run_coroutinet   get_datat   update_datat   delete_datat	   terminatet   pingt   upload_filet   start_ipythont
   call_stackR   t   profile_metadatat   get_logst   keyst   versionst   actor_executet   actor_attributet   closes   compute-taskt   reports   release-tasks   delete-datas   steal-requestt   handlerst   stream_handlerst   io_loopt   connection_argst	   schedulert   ioloopi  t	   heartbeatt   loggers   dask-worker [not started]s#   distributed.worker.profile.intervals   profile-cycle(   RG   RH   (   RG   RR   (   RG   RS   (   RH   RI   (   RH   RR   (   RJ   RI   (   RI   RR   (   RI   RS   (   RI   RT   (   RI   s   long-running(   s   long-runningRS   (   s   long-runningRR   (   s   long-runningRT   (   RG   RU   (   RG   RR   (   RU   RG   (   RU   RR   (   t   dictt   taskst
   task_statet	   dep_statet   dependenciest
   dependentst   waiting_for_datat   who_hasR   t   sett   has_whatR   t   pending_data_per_workert	   threadingt   Lockt   _lockt   data_neededt   in_flight_taskst   in_flight_workerst   daskt   configt   gett   total_out_connectionst   total_in_connectionst   total_comm_nbytest   comm_nbytest   suspicious_depst   _missing_dep_flightt   nbytest   typest   threadst
   exceptionst
   tracebackst   active_threads_lockt   active_threadsR   t   createt   profile_keyst   profile_keys_historyt   profile_recentt   profile_historyt
   prioritiest
   generationt	   durationst   listt
   startstopst   resource_restrictionsRH   RJ   RI   t   executed_countt   long_runningt   Nonet   batched_streamt   recent_messages_logt   target_message_sizet   logt   popt   FalseRQ   t   transition_waiting_readyt   transition_waiting_donet   transition_ready_executingt   transition_ready_memoryt    transition_constrained_executingt   transition_executing_donet!   transition_executing_long_runningt   _transitionst   transition_dep_waiting_flightt   transition_dep_waiting_memoryt   transition_dep_flight_waitingt   transition_dep_flight_memoryt   _dep_transitionst   incoming_transfer_logt   incoming_countt   outgoing_transfer_logt   outgoing_countt   outgoing_current_countt   repetitively_busyt   _clientR?   t   _setup_loggingR9   R   t   _portt   _ncorest   ncorest   total_resourcest   copyt   available_resourcest   death_timeoutt   preloadt   preload_argvt   contact_addresst   memory_monitor_intervalt
   extensionsR7   RA   R    t   ost   patht   abspatht
   _workspacet   new_work_dirt   _workdirt   dir_patht	   local_dirR*   t   securityt
   isinstancet   AssertionErrort   get_connection_argsR~   t   get_listen_argst   listen_argst   parse_memory_limitt   memory_limitt   pausedR^   R_   R`   R   t   datat   callablet   tuplet   zictRa   Rb   Rc   t   ImportErrort   joinR
   R(   R'   t   intt   floatt   weightt   actorsR   t   currentt   loopt   statusR   t   _closedt	   reconnectR,   t   executort   actor_executort   namet   scheduler_delayt   stream_commst   heartbeat_activet   _ipython_kernelt   syst   insertt   servicest   service_portst   service_specst   metricst   low_level_profilerRh   Ri   Rj   Rk   Rl   Rm   Rn   R   Rp   Rq   t   get_call_stackt   get_profilet   get_profile_metadataRt   Ru   Rv   Rw   Rx   Ry   t   add_taskt   release_keyt   steal_requestt   superRK   t   __init__t   rpcR   RY   t   execution_stateR=   R   R}   t   periodic_callbackst   _addresst   _memory_monitoringt   memory_monitort   DEFAULT_EXTENSIONSRD   R   t   _throttled_gcR$   t   trigger_profilet   cycle_profilet   _global_workerst   appendt   weakreft   ref('   t   selft   scheduler_ipt   scheduler_portt   scheduler_fileR   R   R   R  R  R   R   R   R   t	   resourcest   silence_logsR   R   R   R   R   R   R   R  R   R  t   kwargsRV   t   cfgt   scheduler_addrRa   Rb   Rc   R   Rd   t   targetR{   R|   t   pct   ext(    (    s1   lib/python2.7/site-packages/distributed/worker.pyR    s   								














					
!					
					 										


















	
				
c         C` s_   d |  j  j |  j |  j t |  j  t |  j  |  j t |  j  t |  j	  t |  j
  f	 S(   NsJ   <%s: %s, %s, stored: %d, running: %d/%d, ready: %d, comm: %d, waiting: %d>(   t	   __class__t   __name__RY   R   t   lenR   RI   R   RH   R   R   (   R  (    (    s1   lib/python2.7/site-packages/distributed/worker.pyt   __repr__]  s    	c         C` sm   t  d t j j d   |  _ |  j j t j t j j d    t j	 |  j  t
 |  t j |  j  d  S(   Nt   ns   distributed.admin.log-lengths   distributed.admin.log-format(   R<   R   R   R   t   _deque_handlert   setFormattert   loggingt	   FormatterR   t
   addHandlerR   t   removeHandler(   R  (    (    s1   lib/python2.7/site-packages/distributed/worker.pyR   m  s    	c         C` s   |  j  S(   s"    For API compatibility with Nanny (   RY   (   R  (    (    s1   lib/python2.7/site-packages/distributed/worker.pyt   worker_addressw  s    c      
   ` s}   t  d t   j  d t   j  d t   j  d t   j   }   f d     j j   D } t |   j	 j
   |  S(   NRI   t	   in_memoryRH   t	   in_flightc         ` s%   i  |  ] \ } } |    |  q S(    (    (   t   .0t   kt   metric(   R  (    s1   lib/python2.7/site-packages/distributed/worker.pys
   <dictcomp>  s   	 (   R   R+  RI   R   RH   R   R  t   itemsR   t   monitort   recent(   R  t   coret   custom(    (   R  s1   lib/python2.7/site-packages/distributed/worker.pyt   get_metrics|  s    c         C` s?   i t  |   j d 6|  j d 6|  j j d 6|  j d 6|  j d 6S(   Nt   typet   idR   R   R   (   R@  R*  RA  R   RY   R   R   (   R  R   (    (    s1   lib/python2.7/site-packages/distributed/worker.pyt   identity  s    

c          c` s  |  j  d j   t   } |  j d  k r8 |  j |  _ n  t j d d  xMt r|  j	 r t   | |  j	 k r |  j
 d d  Vd  S|  j d0 k r t j  n  yt   } d   |  j j   D } t |  j j d	 |  j V} d
 | _ t j |   | _ | j t d d d t d |  j d t |  j  d |  j d |  j d |  j d | d t   d |  j d |  j d |  j d |  j  d t! j"   d |  j#    d d g V| j$ d d g  } |  j	 r|  j	 t   | } | d k  rwL n  t j% t& d |  |  } n  | V} t   } | | d  }	 | d! |	 |  _' d" |  _ PWqL t( k
 rtt j d# |  j j  t j) d$  VqL t j* k
 rt j d%  qL XqL W| d& d' k rt+ d( | f   n xb | d) D]V }
 t, j- |
  } t. | d*  r | d* |   } n	 |   } t j d+ | | f  qWt j d, |  j j  t j d d  t/ d- d. d/ |  j0  |  _1 |  j1 j2 |  |  j  d j2   |  j0 j3 |  j4 |  d  S(1   NR   t   -i1   t   timeouti   t   closedt   closingc         S` s%   i  |  ] \ } } t  |  |  q S(    (   R/   (   R7  R8  t   v(    (    s1   lib/python2.7/site-packages/distributed/worker.pys
   <dictcomp>  s   	 R~   s   Worker->Schedulert   ops   register-workert   replyRY   Ru   R   R   R   R   t   nowR!  R   t   local_directoryR  t   pidR  t   serializerst   msgpackt   deserializersi    t   secondsi   R!   t   runnings   Waiting to connect to: %26sg?s&   Timed out when connecting to schedulerR   t   OKs%   Unexpected response from register: %rs   worker-setupst   dask_workers   Init function %s ran: output=%ss           Registered to: %26st   intervalt   2msR   (   RE  RF  (5   R  t   stopR!   R   R   RY   R   t   infot   TrueR   Ry   R   R   R   R   R:  R   R   R~   R   R  R  t   _servert   writeR   R   R   R   R   R   R   R   R  R   t   getpidR?  t   readt   with_timeoutR   R   t   EnvironmentErrort   sleept   TimeoutErrort
   ValueErrorR%   t   loadsR1   R   R   R   t   startt   add_callbackt   handle_scheduler(   R  Rc  t   _startR   R   t   futuret   difft   responset   _endt   middlet   function_bytest   setup_functiont   result(    (    s1   lib/python2.7/site-packages/distributed/worker.pyt   _register_with_scheduler  s    																	c         c` s  |  j  s t |  _  t j d |  j  z y t   } |  j j d |  j d t   d |  j	    V} t   } | | d } | d d k r |  j
   Vd  S| d | |  _ | d	 d
 |  j d _ Wn t k
 r t j d  n XWd  t |  _  Xn t j d  d  S(   Ns   Heartbeat: %sRY   RJ  R  i   R   t   missingR!   s   heartbeat-intervali  R   s   Heartbeat to scheduler faileds   Heartbeat skipped: channel busy(   R   RX  R   t   debugRY   R!   R   t   heartbeat_workerR   R?  Ro  R   R  t   callback_timeR   t   warningR   (   R  Rc  Ri  t   endRk  (    (    s1   lib/python2.7/site-packages/distributed/worker.pyR     s&    		 	"	c         c` s   zN y$ |  j  | d |  j |  j g VWn# t k
 rL } t j |    n XWd  |  j r} t j d  |  j j	 |  j
  n |  j d t  VXd  S(   Nt   every_cycles0   Connection to scheduler broken.  Reconnecting...Rz   (   t   handle_streamt   ensure_communicatingt   ensure_computingt	   ExceptionR   t	   exceptionR   RW  R   Rd  Ro  Ry   R   (   R  R   t   e(    (    s1   lib/python2.7/site-packages/distributed/worker.pyRe    s     	c         C` sW   d d l  m } |  j d k rJ | d |  j d i |  d 6d t  |  _ n  |  j j   S(   sU   Start an IPython kernel

        Returns Jupyter connection info dictionary.
        i   (   Rq   t   ipt   nsR]   R   N(   t   _ipython_utilsRq   R   R   R}  R   t   get_connection_info(   R  R   Rq   (    (    s1   lib/python2.7/site-packages/distributed/worker.pyRq     s
    %c         #` s   t  j j |  j |      f d   } t |  d k  rH | |  } n t | |  V} | r y t    Wq t k
 r } t j	 |  t
 j i d d 6t |  d 6  q Xn  t
 j i d d 6t |  d 6  d  S(   Nc         ` sQ   t  |  t  r |  j   }  n  t   d   } | j |   | j   Wd  QX|  S(   Nt   wb(   R   R   t   encodet   openRZ  t   flush(   R   t   f(   t   out_filename(    s1   lib/python2.7/site-packages/distributed/worker.pyt   func   s    i'  RS   R   R{  RR  R   (   R   R   R   R   R+  R   R6   Rz  R   R{  R   R   R&   (   R  R   t   filenameR   t   loadR  R|  (    (   R  s1   lib/python2.7/site-packages/distributed/worker.pyRp     s    *c         C` s   t  |  j  S(   N(   R   R   (   R  R   (    (    s1   lib/python2.7/site-packages/distributed/worker.pyRu   6  s    c         #` s     f d   | j    D } t | d   j d   j V\ } } } | r} t j d | | |  t i d d 6| d 6  n)   j d | d	 t  t i d
 d 6  d  S(   Nc         ` sG   i  |  ]= \ } } |   j  k r g  | D] } t |  ^ q( |  q S(    (   R   R   (   R7  R8  RG  t   addr(   R  (    s1   lib/python2.7/site-packages/distributed/worker.pys
   <dictcomp><  s   		R  t   whos4   Could not find data: %s on workers: %s (who_has: %s)s   missing-dataR   Ru   R   Rz   RR  (	   R:  RC   R  RY   R   Rt  R   Rl   R   (   R  R   R   Rn  t   missing_keyst   missing_workers(    (   R  s1   lib/python2.7/site-packages/distributed/worker.pyRh   9  s    %c         C` s   |  j  } | d  k r' t | j  } n< | j } g  t t | t |    D] } | | ^ qL } g  | D] } | j | j |  f ^ qj S(   N(	   R.  R   R   R   t   ranget   minR+  t	   levelnamet   format(   R  R   R-  t   deque_handlert   Lt   it   msg(    (    s1   lib/python2.7/site-packages/distributed/worker.pyRt   O  s    		3c         C` sC  | d k r d } n  x'|  j  j   D]\ } } d  } t | t  rU | \ } } n d } t | t t f  r | j d  } n  t | t t f  r | d t	 | d  } } n  t | t  r | \ } } n i  } | |  d |  j
 | |  j | <|  j | j | d  k	 r| n | | f  |  j | j |  j | <q% Wd  S(   Ns   0.0.0.0RO   i    t   :i   R}   (   R  R:  R   R   R   t   strR   t   splitR   R   R   R  t   listent   portR  (   R  t   default_listen_ipR8  RG  t	   listen_ipR  R#  (    (    s1   lib/python2.7/site-packages/distributed/worker.pyt   start_services\  s$    	i    c         c` s  |  j  d  k s t  t   t t _ | st t |  j j	  } |  j
 t |  j j	  d |  j t |  j	  |  _ n t | t  r t t |  j j	   } |  _ |  j
 | | f d |  j nR |  j
 | d |  j t |  j	  |  _ y t |  } Wn t k
 r| } n Xd | k r7| j d  \ } } n  |  j pF|  j	 |  _ t |  j d |  d |  j d |  j |  j |  y  d |  j j | |  j f } Wn' t k
 rd |  j j | f } n Xt j d |  j	  t j d	 |  xB |  j j    D]1 \ } } t j d
 | | d t! |  f  qWt j d |  j j	  t j d d  t j d |  j"  |  j# rt j d t$ |  j#   n  t j d |  j  t% d |  j	  |  j&   V|  j'   t( j) |    d  S(   NR   s   ://t	   parametert   file_dirt   argvs   %s%s:%ds   %s%ss         Start worker at: %26ss            Listening to: %26ss     %16s at: %26sR  s   Waiting to connect to: %26sRC  i1   s                 Threads: %26ds                  Memory: %26ss         Local Directory: %26ss   dask-worker [%s](*   R   R   R   RE   RX  R8   t   on_event_loop_threadR   R   RY   R  R   R   R}  R   R   R0   Ra  R  R   R#   R   R   R   R  t   listenerR\   R  Rz  R   RW  R  R:  R  R   R   R;   R$   Ro  t   start_periodic_callbacksR   R   (   R  t   addr_or_portt   listen_hostt   protocolt   listening_addressR8  RG  (    (    s1   lib/python2.7/site-packages/distributed/worker.pyRf  x  s`    	

		
	)	
c         C` s   |  j    j   S(   N(   Rf  t	   __await__(   R  (    (    s1   lib/python2.7/site-packages/distributed/worker.pyR    s    c         C` s   |  j  j |  j |  d  S(   N(   R   Rd  Rf  (   R  R  (    (    s1   lib/python2.7/site-packages/distributed/worker.pyRc    s    c         O` s#   t  j d d d |  j | |   S(   Ns'   Worker._close has moved to Worker.closet
   stackleveli   (   t   warningst   warnRy   (   R  t   argsR#  (    (    s1   lib/python2.7/site-packages/distributed/worker.pyt   _close  s    i
   c   
      c` s  t    |  j d k r d  St   y t j d |  j  Wn t k
 r[ t j d  n Xd |  _ t d  |  j   x! |  j	 j
   D] } | j   q Wt t t j  : | r t j t d |  |  j j d |  j   Vn  Wd  QX|  j j   |  j j j j   t |  j t  rL|  j j j j   |  j j d | d	 |  n |  j j d t  |  j j d | d	 |  |  j j   x' |  j  j!   D] \ } } | j   qW|  j" r|  j" j# j$   r|  j" j% i d
 d 6 n  |  j" r|  j" j&   n  | r\d |  j' k r\d |  j( j) |  j* |  j' d f } |  j+ |   }	 |	 j,   VWd  QXn  |  j+ j&   |  j- j.   |  j/   d |  _ t0 j& |   Vt d  Wd  QXd  S(   NRE  RF  s   Stopping worker at %ss   Stopping workers   dask-worker [closing]RP  RY   t   waitRD  s   close-streamRH  t   nannys   %s%s:%ds   dask-worker [closed](   RE  RF  (1   R3   R   RF   R   RW  RY   Ra  R$   RV  R  t   valuesR4   R^  R   R`  R]  R   R   t
   unregisterR   t	   close_rpcR   t   _work_queuet   queuet   clearR   R   R,   t   shutdownR   R   t   releaseR  R:  R   R   RE  t   sendRy   R  R  R\   R}  R  Rn   R   R   t   _remove_from_global_workersR"   (
   R  Rz   RD  R  t   executor_waitR'  R8  RG  t   nanny_addresst   r(    (    s1   lib/python2.7/site-packages/distributed/worker.pyRy     sZ    
	

#		
	c         C` s   |  j    d  S(   N(   R  (   R  (    (    s1   lib/python2.7/site-packages/distributed/worker.pyt   __del__  s    c         C` sY   xR t  t  D]D } |   |  k r2 t j |  n  |   d  k r t j |  q q Wd  S(   N(   R   R  t   removeR   (   R  R  (    (    s1   lib/python2.7/site-packages/distributed/worker.pyR    s
    c         c` s!   |  j  d |  Vt d   d  S(   NRz   RR  (   Ry   R   (   R  R   Rz   (    (    s1   lib/python2.7/site-packages/distributed/worker.pyRn     s    c         c` s'   |  j  j   V|  j d k s# t  d  S(   NRE  (   R   R  R   R   (   R  (    (    s1   lib/python2.7/site-packages/distributed/worker.pyt   wait_until_closed  s    c         ` s}      j  k re t d d d  j     j    <t j     f d    }  j j |  n   j    j |  d  S(   NRT  t   1msR   c          3` sE   t    d  j V}  d |  _ |  j i d d 6 V j |   d  S(   NR~   s   Worker->Workert   connection_streamRH  (   R   R~   R   RZ  Rc  (   R   (   RY   t   bcommR  (    s1   lib/python2.7/site-packages/distributed/worker.pyt   batched_send_connect"  s
    	(   R   R   R   R   t	   coroutineRd  R  (   R  RY   R  R  (    (   RY   R  R  s1   lib/python2.7/site-packages/distributed/worker.pyt   send_to_worker  s    
c         #` s-  t    } | d  k r!   j } n  | rX | rX t | j  t   j  k rX | d } n  | t k	 r   j | k r t j	 i d d 6  n    j d 7_   f d   | D } t
 |  t
 |  k  r5xi t |  t |  D]N } |   j k r d d l m }	 |	 t   j |    j |  | | <q q Wn  i d d 6d   | j   D d	 6}
   f d
   | D } t    }   j d  k	 r  j d j | |  n  t    } z yE | j |
 d | V} | j d |  V} | d k st |   Wn: t k
 r/t j d   j | d t | j     n XWd    j d 8_ Xt    }   j d  k	 rw  j d j | |  n  t t d  | j     }   j d 7_ | | pd }   j j  i	 |   j! d 6|   j! d 6| | d d 6| d 6| d 6| d 6| d 6| d 6| | d 6 t j	 d   d  S(   Ni   t   busyR   i   c         ` s/   i  |  ]% } |   j  k r   j  | |  q S(    (   R   (   R7  R8  (   R  (    s1   lib/python2.7/site-packages/distributed/worker.pys
   <dictcomp>H  s   	 (   t   ActorRR  c         S` s%   i  |  ] \ } } t  |  |  q S(    (   R&   (   R7  R8  RG  (    (    s1   lib/python2.7/site-packages/distributed/worker.pys
   <dictcomp>Q  s   	 R   c         ` s%   i  |  ] }   j  j |  |  q S(    (   R   R   (   R7  R8  (   R  (    s1   lib/python2.7/site-packages/distributed/worker.pys
   <dictcomp>R  s   	 s   get-data-load-durationRM  RO  s$   failed during get data with %s -> %st   exc_infos   get-data-send-durationg      ?Rc  RV  Rk  t   durationR  Ru   t   totalt
   compressedt	   bandwidths
   dont-reply("   R!   R   R   R   t   peer_addressRY   R   R   R   R   R+  R   R   t   actorR  R@  R:  t   digestst   addRZ  R\  R   R^  R   R{  RX  t   abortt   sumt   filterR  R   R   R  R   (   R  R   Ru   R  RM  t   max_connectionsRc  R   R8  R  R  R   RV  R  Ri  t   total_bytesR  (    (   R  s1   lib/python2.7/site-packages/distributed/worker.pyRk   0  sh    	0$		 
		c         C` s,  x | j    D] \ } } | |  j k rA |  j | d d | nT |  j | |  d |  j | <d  |  j | <d  |  j | <d  |  j | <t   |  j	 | <| |  j
 k r |  j | d d | n  |  j j | d f  q W| r|  j j i d d 6t |  d 6 n  i d   | j    D d 6d	 d
 6} | S(   NRR   t   values   receive-from-scatters   add-keysRH  Ru   c         S` s%   i  |  ] \ } } t  |  |  q S(    (   t   sizeof(   R7  R8  RG  (    (    s1   lib/python2.7/site-packages/distributed/worker.pys
   <dictcomp>  s   	 R   RR  R   (   R:  R   t
   transitiont   put_key_in_memoryR   R   R   R   R   R   R   t   transition_depR   R  R   R  R   (   R  R   R   Rz   RM  t   keyR  RW  (    (    s1   lib/python2.7/site-packages/distributed/worker.pyRl     s     '$c         c` s   | r xh t  |  D]Z } |  j j | d f  | |  j k rN |  j |  n  | |  j k r |  j |  q q Wt j d t	 |   | r t j d  |  j
 j d |  j d t  |   Vq n  t d   d  S(   Nt   deletes   Deleted %d keyss#   Reporting loss of keys to schedulerRY   Ru   RR  (   R   R   R  R   R  R   t   release_depR   Rq  R+  R   t   remove_keysR   R   (   R  R   Ru   Rz   R  (    (    s1   lib/python2.7/site-packages/distributed/worker.pyRm     s    c         k` s   xd | j    D]V \ } } | |  j k rI |  j | c | |  j | 7<n | |  j | <| |  j | <q W|  j j d |  j d |  j  Vd  S(   NR!  R]   (   R:  R   R   R   t   set_resourcesR   (   R  R!  R  t   quantity(    (    s1   lib/python2.7/site-packages/distributed/worker.pyR    s    !c         ` sX  y|   j  k r   j | } | d k rs |   j k sO |   j k sO t  t j d | |    j |  d  S| t k r d  S| d k r   j	 | =  j
 | =q n  | d  k	 r t |    j f }   j d 8_ n    j j |  d k rPd   j | <  j |  d    j  | <  j j | d f  |   j | <|	   j | <d  S  j j | d f  yx t   } t | | | |    j  | <| rd    j | <n  t   } | | d k r  j | j d | | f  n  Wnm t k
 rM} t j d	 d
 t t |  } | | d <d | d <  j j |    j j | d f  d  SX|   j | <|	   j | <|
 r~|
   j | <n  d   j | <| d  k	 r  j j |  n  | pi  } t  |    j! | <t      j" | <x | D] } |   j# k rt      j# | <n    j# | j$ |  |   j k r{  j j |  d k rLd } n d } |   j | <  j j | d | f  n    j | d k r  j" | j$ |  qqWx | j%   D] \ } } | st  |   j& k rt  |    j& | <n    j& | j |  xL | D]D }   j' | j$ |    j | d k r  j( | j |  qqWqW  j" | rz  j) j |  n   j* | d    j+ r| rt,   f d   | D  st  t,   f d   | D  st  x | D] }   j- |  qW  j. |  qn  WnB t k
 rS} t j/ |  t0 rMd d  l1 } | j2   n    n Xd  S(   NRR   s,   Asked to compute pre-existing result: %s: %st   erredi   s   new-task-already-in-memoryt   newg{Gz?t   deserializes   Could not deserialize taskR  R  s
   task-erredRH  s   deserialize-errorRG   s   new-depRH   c         3` s   |  ] } |   j  k Vq d  S(   N(   R   (   R7  t   dep(   R  (    s1   lib/python2.7/site-packages/distributed/worker.pys	   <genexpr>(  s    c         3` s   |  ] } |   j  k Vq d  S(   N(   R   (   R7  R  (   R  (    s1   lib/python2.7/site-packages/distributed/worker.pys	   <genexpr>)  s    i    (3   R   R   R   R   R   R   Rq  t   send_task_state_to_schedulert   IN_PLAYR   R   R   R   R   R   R   R   R  R   R   R!   t   _deserializeR   Rz  Rt  RX  R   R   R  R   R   t   updateR   R   R   R   R  R:  R   R   R   R   R  RQ   t   allt   validate_dept   validate_keyR{  t   LOG_PDBt   pdbt	   set_trace(   R  R  t   functionR  R#  t   taskR   R   t   priorityR  R   R  t   kwargs2t   stateRc  RV  R|  t   emsgR  t   workersR]   R  (    (   R  s1   lib/python2.7/site-packages/distributed/worker.pyR
    s    $
		$

		""c         K` s   y |  j  | } Wn t k
 r% d  SX| | k r6 d  S|  j | | f } | | |  } |  j j d | | | ps | f  | |  j  k r | p | |  j  | <|  j r |  j |  q n  d  S(   NR  (   R   t   KeyErrorR   R   R  RQ   R  (   R  R  t   finishR#  Rc  R  R  (    (    s1   lib/python2.7/site-packages/distributed/worker.pyR  5  s    "	c         C` s   yE |  j  r7 | |  j k s! t  |  j | s7 t  n  | |  j | <WnB t k
 r } t j |  t r d d  l } | j	   n    n Xd  S(   Ni    (
   RQ   R   R   R   Rz  R   R{  R  R  R  (   R  R  R]   R|  R  (    (    s1   lib/python2.7/site-packages/distributed/worker.pyR   D  s    	c         C` s  yY|  j  r$ | |  j k s$ t  n  |  j | =| r y |  j | j |  Wn t k
 r_ n Xy |  j | j |  Wq t k
 r q Xn  |  j j |  s | |  j k r |  j j	 |  |  j
 j |  j |  q n  x\ |  j j | d  D]E } |  j | d k r | r$|  j j |  q7|  j j |  q q W|  j | sX|  j |  n  WnB t k
 r} t j |  t rd d  l } | j   n    n Xd  S(   NRG   i    (    (   RQ   R   R   R   R  R  R   R   R   R  R   Rd  t   handle_missing_depR   R   R   t
   appendleftR  R  Rz  R   R{  R  R  R  (   R  R  R]   R  R  R|  R  (    (    s1   lib/python2.7/site-packages/distributed/worker.pyR   S  s<    	
c         C` s   y |  j  r$ | |  j k s$ t  n  |  j | =|  j | r| d |  j | <|  j | |  |  j j i d d 6| g d 6 n |  j |  WnB t	 k
 r } t
 j |  t r d d  l } | j   n    n Xd  S(   NRR   s   add-keysRH  Ru   i    (   RQ   R   R   R   R   R  R   R  R  Rz  R   R{  R  R  R  (   R  R  R  R|  R  (    (    s1   lib/python2.7/site-packages/distributed/worker.pyR   x  s    	
$c         C` s   yh |  j  rg | |  j k s! t  | |  j k s6 t  | |  j k sK t  |  j | d k sg t  n  WnB t k
 r } t j |  t	 r d d  l
 } | j   n    n X| t k	 r | |  j k r |  j | | d t n  d  S(   NRR   i    R  (   RQ   R   R   R   R   R   Rz  R   R{  R  R  R  t   no_valueR  R   (   R  R  R  R|  R  (    (    s1   lib/python2.7/site-packages/distributed/worker.pyR     s    	 c         K` s   |  j  | } | | k r d  S|  j | | f } | | |  } |  j j | | | pW | f  | pg | |  j  | <|  j r |  j |  n  d  S(   N(   R   R   R   R  RQ   R  (   R  R  R  R#  Rc  R  R  (    (    s1   lib/python2.7/site-packages/distributed/worker.pyR    s    	c         ` sD  y   j  r   j | d k s% t  |   j k s: t    j | sN t  t   f d     j | D  sw t  |   j k s t  |   j k s t  n    j j | d   |   j
 k r   j j |  d St j   j   j | | f  WnB t k
 r?} t j |  t r9d d  l } | j   n    n Xd  S(   NRG   c         3` s-   |  ]# } |   j  k p$ |   j k Vq d  S(   N(   R   R   (   R7  R  (   R  (    s1   lib/python2.7/site-packages/distributed/worker.pys	   <genexpr>  s   RJ   i    (   RQ   R   R   R   R  R   RI   RH   R   R   R   RJ   R  t   heapqt   heappushR   Rz  R   R{  R  R  R  (   R  R  R|  R  (    (   R  s1   lib/python2.7/site-packages/distributed/worker.pyR     s*    	$c         C` s   y |  j  rg |  j | d k s% t  | |  j k s: t  | |  j k sO t  | |  j k sg t  n  |  j | =|  j |  WnB t k
 r } t j	 |  t
 r d d  l } | j   n    n Xd  S(   NRG   i    (   RQ   R   R   R   RI   RH   R  Rz  R   R{  R  R  R  (   R  R  R  R|  R  (    (    s1   lib/python2.7/site-packages/distributed/worker.pyR     s    	
c         ` s   y   j  r{ |   j k s! t    j | t k s: t  |   j k sO t  t   f d     j | D  s{ t  n    j j	 |    j
 j   j |  WnB t k
 r } t j |  t r d d  l } | j   n    n Xd  S(   Nc         3` s-   |  ]# } |   j  k p$ |   j k Vq d  S(   N(   R   R   (   R7  R  (   R  (    s1   lib/python2.7/site-packages/distributed/worker.pys	   <genexpr>  s   i    (   RQ   R   R   R   t   READYRH   R  R   RI   R  R   Rd  t   executeRz  R   R{  R  R  R  (   R  R  R|  R  (    (   R  s1   lib/python2.7/site-packages/distributed/worker.pyR     s     	c         C` s   |  j  |  d  S(   N(   R  (   R  R  R  (    (    s1   lib/python2.7/site-packages/distributed/worker.pyR     s    c         C` sy   |  j  |  x4 |  j | j   D] \ } } |  j | c | 8<q! W|  j ru t d   |  j j   D  su t  n  d  S(   Nc         s` s   |  ] } | d  k Vq d S(   i    N(    (   R7  RG  (    (    s1   lib/python2.7/site-packages/distributed/worker.pys	   <genexpr>  s    (   R   R   R:  R   RQ   R  R  R   (   R  R  t   resourceR  (    (    s1   lib/python2.7/site-packages/distributed/worker.pyR     s
     	c   
      C` sY  y|  j  r] | |  j k s0 | |  j k s0 t  | |  j k sE t  | |  j k s] t  n  d  } | |  j k r x7 |  j | j   D] \ } } |  j	 | c | 7<q Wn  |  j
 | d k r |  j j |  |  j d 7_ n& |  j
 | d k r|  j j |  n  | t k	 ry' d |  j
 | <|  j | | d t Wng t k
 r} t j d d t t |  } | d |  j | <| d	 |  j | <d
 |  j
 | <d
 } n X| |  j k r|  j | d  qn  | r|  j r|  j |  n t  | SWn_ t k
 rt j d  nB t k
 rT} t j |  t rNd d  l }	 |	 j    n    n Xd  S(   NRI   i   s   long-runningRR   R  s   Failed to put key in memoryR  R{  t	   tracebackRS   s   Comm closedi    (!   RQ   RI   R   R   R   RH   R   R   R:  R   R   R  R   R  R  R   Rz  R   RW  RX  R   R   R   R   R  R   R  R   R^  R{  R  R  R  (
   R  R  R  Rz   t   outR  R  R|  R  R  (    (    s1   lib/python2.7/site-packages/distributed/worker.pyR     sN    	$ 
c         C` s   yt |  j  r$ | |  j k s$ t  n  |  j j |  |  j j |  |  j j i d d 6| d 6| d 6 |  j   WnB t	 k
 r } t
 j |  t r d d  l } | j   n    n Xd  S(   Ns   long-runningRH  R  t   compute_durationi    (   RQ   RI   R   R  R   R  R   R  Ry  Rz  R   R{  R  R  R  (   R  R  R  R|  R  (    (    s1   lib/python2.7/site-packages/distributed/worker.pyR   *  s    		c         C` s5   |  j  j |  d k r1 |  j | d d | n  d  S(   NRI   s   long-runningR  (   R   R   R  (   R  R  R  (    (    s1   lib/python2.7/site-packages/distributed/worker.pyt   maybe_transition_long_running>  s    c         C` sM   i | |  j  k d 6| |  j k d 6| t d |  j  k d 6| |  j k d 6S(   NRI   R   i   t   heapR   (   RI   R   R	   RH   R   (   R  R  (    (    s1   lib/python2.7/site-packages/distributed/worker.pyt   stateofB  s
    c         ` sR   g  |  j  D]D   t   f d   | D  sH t   f d   | D  r
   ^ q
 S(   Nc         3` s   |  ] } |   k Vq d  S(   N(    (   R7  R  (   R  (    s1   lib/python2.7/site-packages/distributed/worker.pys	   <genexpr>N  s    c         3` s@   |  ]6 }   D]) } t  | t t t f  r | | k Vq q d  S(   N(   R   R   R   R   (   R7  R  t   c(   R  (    s1   lib/python2.7/site-packages/distributed/worker.pys	   <genexpr>P  s   
(   R   t   any(   R  Ru   (    (   R  s1   lib/python2.7/site-packages/distributed/worker.pyt   storyJ  s    c         ` s  t  } yx| r  j rt   j    j k  rt } t j d t   j  t   j    j    j d } |   j k r   j j	   t  } q n    j
 j |  d k r   j j | d f    j j	   t  } q n    j | }   j rt   f d   | D  st  n  g  | D] }   j | d k r"| ^ q"}   f d   | D } | rt j d |    f d   | D } x | D] }   j j |  qW  j j   j |  g  | D] } | | k r| ^ q} n    j j d	 | | f  t } x| rt   j    j k  s:  j   j k  r| j   }   j | d k r_q
n  |   j k rtq
n  g    j | D] } |   j k r| ^ q}	 |	 st  } q
n  t   j  }
 g  |	 D] } t |  |
 k r| ^ q} | rt j  |  } n t j  t! |	   }   j" | |  \ } }   j | 7_ |   j | <x$ | D] }   j# | d
 d | qWW  j j   j$ | | | | d | t  } q
W| r | r   j j	   q q WWnB t% k
 r} t j& |  t' rd d  l( } | j)   n    n Xd  S(   Ns7   Ensure communicating.  Pending: %d.  Connections: %d/%di    RG   s   communication passc         3` s   |  ] } |   j  k Vq d  S(   N(   R   (   R7  R  (   R  (    s1   lib/python2.7/site-packages/distributed/worker.pys	   <genexpr>v  s    c         ` s(   h  |  ] }   j  j |  s |  q S(    (   R   R   (   R7  R  (   R  (    s1   lib/python2.7/site-packages/distributed/worker.pys	   <setcomp>z  s   	 s"   Can't find dependencies for key %sc         ` s%   h  |  ] } |   j  k r |  q S(    (   R   (   R7  R  (   R  (    s1   lib/python2.7/site-packages/distributed/worker.pys	   <setcomp>~  s   	s   gather-dependenciesRU   R]   t   cause(*   RX  R   R+  R   R   R   R   Rq  R   t   popleftR   R   R   R  R   RQ   R  R   R   RW  R   R  R   Rd  R  R   R   R   R   R   RY   t   randomt   choiceR   t   select_keys_for_gatherR  t
   gather_depRz  R{  R  R  R  (   R  t   changedR  t   depsR  t   missing_depst   missing_deps2R6  t   wR  t   hostt   localR]   t	   to_gathert   total_nbytest   dR|  R  (    (   R  s1   lib/python2.7/site-packages/distributed/worker.pyRx  W  s    	
	%,
(	/+	
c         C` s  | |  j  k s! |  j j |  ry |  j  | } Wn t k
 rR |  j | } n X|  j | pi t |  } |  j j |  p t |  } ~ y t |  } Wn# t	 k
 r t
 j | j  } n Xi d d 6d d 6| d 6| d 6|  j j |  d 6| d 6t |  d	 6} nz | |  j k rki d
 d 6d d 6| d 6|  j j |  d 6|  j | d 6|  j | d 6} n t j d | |  j |  d  S| |  j k r|  j | | d <n  |  j j |  d  S(   Ns   task-finishedRH  RR  R   R  R   t   threadR@  R/   s
   task-erredRS   R{  R   s'   Key not ready to send to worker, %s: %sR   (   R   R   R   R  R   R  R   R@  t   dumps_functionR   R%   t   dumpsR*  R   R/   R   R   R   RS   R   R   R   R  (   R  R  R  R   t   typt   typ_serializedR  (    (    s1   lib/python2.7/site-packages/distributed/worker.pyR    sD    !c         C` sw  | |  j  k r d  S| |  j k r2 | |  j | <nO t   } | |  j  | <t   } | | d k r |  j | j d | | f  n  | |  j k r t |  |  j | <n  t |  |  j | <xy |  j	 j
 | d  D]b } | |  j k r | |  j | k r|  j | j |  n  |  j | s1|  j | d  q1q q W| r]| |  j k r]|  j | d  n  |  j j | d f  d  S(   Ng{Gz?s
   disk-writeRH   RR   s   put-in-memory(    (   R   R   R!   R   R  R   R  R@  R   R   R   R   R  R  R   R   (   R  R  R  R  Rc  RV  R  (    (    s1   lib/python2.7/site-packages/distributed/worker.pyR    s*    		 c         C` s   | h } |  j  | } |  j | } xp | r | j   } |  j j |  d k rV q& n  | |  j  | |  j k rt Pn  | j |  | |  j  | 7} q& W| | f S(   NRG   (   R   R   R
  R   R   R   R  (   R  R]   R  R  R  R  R  (    (    s1   lib/python2.7/site-packages/distributed/worker.pyR    s    		c         #` sI    j  d k r d  St   'i  } z~y  j r?   j   n  t   f d   | D  }   j j d | | | f  t j d t	 |   t
   } t   j | | d   j V} t
   } | d d k r*  j j d | | f  x< | D]4 }   j j | d   d	 k r   j | d
  q q Wd  S| r^  j | j d |   j |   j f  n  t   f d   | d D  }	 | | pd }
   j j i |   j d 6|   j d 6| | d   j d 6|
 d 6  f d   | d D d 6|	 d 6|	 |
 d 6| d 6   j d  k	 rI  j d j |	 |
    j d j |
  n    j d j t	 | d     j d 7_   j j d | t | d  f  | d r  j j i d d 6t | d  d 6 n  Wn t k
 rU} t j d |    j j d  | f  x   j j  |  D]4 }   j! | j" |    j! | s  j! | =qqWnK t# k
 r} t j |    j rt$ rd! d  l% } | j&   n    n XWd    j' | 8_' | j d d"  d k } | j d i   } x   j( j  |  D] } | r&| | k r&  j | d# d$ | | n8   j j |  d# k r^  j | d
 d% | d& | n  | r| | k r|   j) k r  j j d' | f    j j i d( d 6| d) 6| d* 6 qqW  j r  j   n    j*   | sd!   _+   j,   n@   j+ d 7_+ t- j. d+ d,   j+  V  j/ |  V  j,   XWd  QXd  S(-   NRQ  c         3` s-   |  ]# }   j  j |  d  k r | Vq d S(   RU   N(   R   R   (   R7  R  (   R  (    s1   lib/python2.7/site-packages/distributed/worker.pys	   <genexpr>  s    s   request-deps   Request %d keysR  R   R  s   busy-gatherRU   RG   t   transferc         3` s$   |  ] }   j  j | d   Vq d S(   i    N(   R   R   (   R7  R  (   R  (    s1   lib/python2.7/site-packages/distributed/worker.pys	   <genexpr>7  s    R   g      ?Rc  RV  g       @Rk  R  c         ` s(   i  |  ] }   j  j | d   |  q S(   N(   R   R   R   (   R7  R  (   R  (    s1   lib/python2.7/site-packages/distributed/worker.pys
   <dictcomp>@  s   	 Ru   R  R  s   transfer-bandwidths   transfer-durations   transfer-counti   s   receive-deps   add-keysRH  s+   Worker stream died during communication: %ss   receive-dep-failedi    RO   RR   R  R]   R  s   missing-deps   missing-datat   errant_workerR  g?g      ?(0   R   R3   RQ   t   validate_stateR   R   R  R   Rq  R+  R!   t   get_data_from_workerR  RY   R   R   R   R  R   R   R  R   R  R  t   countersR   R   R   R  R^  R{  R   R   R   R  Rz  R  R  R  R   R   R   Ry  R   Rx  R   R_  t   query_who_has(   R  R]   R  R  R  R	  Ri  Rc  RV  R  R  R|  R  R  R  R   (    (   R  s1   lib/python2.7/site-packages/distributed/worker.pyR    s    
 			
 	#
	&"	#	
	c         C` s}   t  d t |   } xS |  j | D]D } t |  } | d |  j | <| d |  j | <|  j | d  q$ W|  j |  d  S(   Ns/   Could not find dependent %s.  Check worker logsR{  R   RS   (   Ra  R  R   R   R   R   R  R  (   R  R  t   excR  R  (    (    s1   lib/python2.7/site-packages/distributed/worker.pyt   bad_dep  s    c   	      /` s  t  |  }   j j d | f  z(y  f d   | D } | sH d  SxJ t  |  D]< }   j | } | d k rU | j |    j |  qU qU W| s d  Sx( | D]  } t j d |   j |  q W  j j	 d t  |   V} d   | j
   D }   j |  x | D] }   j | c d 7<| j |  sm  j j | d   j j |  f    j |  q  j j | d	 f  x?   j j | d  D]( } |   j k r  j j |  qqWqWWn| t k
 rHt j d
 d t | j d d  }   j j d | | f  | d k rB  j   d | d | VqI  n XWd  y% x | D] }   j j |  qWWWn t k
 rn X  j   Xd  S(   Ns   handle-missingc         ` s%   h  |  ] } |   j  k r |  q S(    (   R   (   R7  R  (   R  (    s1   lib/python2.7/site-packages/distributed/worker.pys	   <setcomp>  s   	 i   s.   Dependent not found: %s %s .  Asking schedulerRu   c         S` s%   i  |  ] \ } } | r | |  q S(    (    (   R7  R8  RG  (    (    s1   lib/python2.7/site-packages/distributed/worker.pys
   <dictcomp>  s   	 i   s   no workers founds   new workers founds#   Handle missing dep failed, retryingR  t   retriess   handle-missing-failedi    (    (   R   R   R  R   R  R%  R   RW  R   R   R:  t   update_who_hasR   R   R  R   R   Rz  RS   RX  R  R   R  Rx  (	   R  R  R#  t   original_depsR  t
   suspiciousR   R  R&  (    (   R  s1   lib/python2.7/site-packages/distributed/worker.pyR    sX     %c         g` sF   t    7 |  j j d |  V} |  j |  t j |   Wd  QXd  S(   NRu   (   R3   R   R   R'  R   R   (   R  R  Ri  (    (    s1   lib/python2.7/site-packages/distributed/worker.pyR#    s    
c         C` s   y x | j    D]v \ } } | s( q n  | |  j k rN |  j | j |  n t |  |  j | <x" | D] } |  j | j |  qh Wq WWnB t k
 r } t j |  t	 r d d  l
 } | j   n    n Xd  S(   Ni    (   R:  R   R  R   R   R  Rz  R   R{  R  R  R  (   R  R   R  R  R]   R|  R  (    (    s1   lib/python2.7/site-packages/distributed/worker.pyR'    s     c         C` s`   |  j  j | d   } i d d 6| d 6| d 6} |  j j |  | d k r\ |  j |  n  d  S(   Ns   steal-responseRH  R  R  RH   RG   (   RH   RG   (   R   R   R   R   R  R  (   R  R  R  Ri  (    (    s1   lib/python2.7/site-packages/distributed/worker.pyR    s
    c         C` sI  y| |  j  k r d  S|  j  j |  } | rQ |  j j | d i | d 6f  n |  j j | d f  |  j | =| |  j k r | |  j k r y |  j | =Wn$ t k
 r t j	 d d t
 n X|  j | =|  j | =n  | |  j k r| |  j k r|  j | =|  j | =|  j | =n  | |  j k r6|  j | =n  xt |  j j | d  D]] } | |  j k rL|  j | j |  |  j | r|  j | d k r|  j |  qqLqLW| |  j k r|  j | =n  |  j | =|  j | =| |  j k r|  j | =n  | |  j k r|  j | =n  | |  j k r1|  j | =n  | |  j k rS|  j j |  n  | |  j k r| d k rx7 |  j | j   D] \ } } |  j | c | 7<qWn  |  j | =n  | r| t k r|  j j  i d d	 6| d
 6| d 6 n  WnR t! k
 rnB t" k
 rD}	 t j# |	  t$ r>d d  l% }
 |
 j&   n    n Xd  S(   Ns   release-keyR	  s$   Tried to delete %s but no file foundR  RG   RU   RI   R  RH  R  i    (    (   RG   RU   ('   R   R   R   R  R   R   R   t   FileNotFoundErrorR   RS   RX  R   R   R   R   R   R   t   discardR  R   R   R   R   R   R   RI   R  R   R:  R   t
   PROCESSINGR   R  R   Rz  R{  R  R  R  (   R  R  R	  t   reasonRz   R  R  R  R  R|  R  (    (    s1   lib/python2.7/site-packages/distributed/worker.pyR    sn    #



 	

 ,c         C` s  y| |  j  k r d  S|  j j | d f  |  j  j |  } | |  j k rZ |  j | =n  | |  j k r x1 |  j j |  D] } |  j | j |  q| Wn  | |  j k r| |  j	 k r |  j	 | =|  j
 | =n  | |  j k r |  j | =|  j
 | =n  |  j | =n  | |  j k r=|  j j |  } |  j | j |  n  xF |  j j | d  D]/ } |  j | d k rS|  j | d | qSqSW| r| d k r|  j j i d d 6| g d 6 n  WnB t k
 r} t j |  t rd d  l } | j   n    n Xd  S(	   Ns   release-depRR   R	  s   release-worker-dataRH  Ru   i    (    (   R   R   R  R   R   R   R   R  R   R   R   R   R   R   R   R   R  R   R  Rz  R   R{  R  R  R  (   R  R  Rz   R  R]   R  R|  R  (    (    s1   lib/python2.7/site-packages/distributed/worker.pyR  /  s@    

(c         C` s1  y |  j  j |  t k r d  S|  j  | =|  j | =| |  j k rO |  j | =n  xK |  j j | d  D]4 } |  j | j |  |  j | se |  j | =qe qe W| |  j k r | |  j	 k r |  j	 | =n  | |  j
 k r |  j
 | =q n  WnB t k
 r,} t j |  t r&d d  l } | j   n    n Xd  S(   Ni    (    (   R   R   t   PENDINGR   R   R   R   R   R  R   R   Rz  R   R{  R  R  R  (   R  R  R  R|  R  (    (    s1   lib/python2.7/site-packages/distributed/worker.pyt   rescind_keyX  s,    

c         #` s   | p |  j  } t d c d 7<| p( i  } | j | | |    t    f d   d  } | j   z	   VWd | j   X  j   } t j |   d S(   sC   Safely run function in thread pool executor

        We've run into issues running concurrent.future futures within
        tornado.  Apparently it's advantageous to use timeouts and periodic
        callbacks to ensure things run smoothly.  This can get tricky, so we
        pull it off into an separate method.
        i    i   c           ` s   t  j d    j  S(   Ns   future state: %s - %s(   R   Rq  t   _state(    (   Rg  R  (    s1   lib/python2.7/site-packages/distributed/worker.pyRN     RO   i  N(	   R   t   job_countert   submitR=   Rc  RV  Rn  R   R   (   R  R  R  R  R#  R   R'  Rn  (    (   Rg  R  s1   lib/python2.7/site-packages/distributed/worker.pyt   executor_submity  s    	
	c         C` s1   | p	 i  } t  |  | d | d | d | d | S(   NR  R  R#  R  (   Ri   (   R  R   R  R  R  R#  (    (    s1   lib/python2.7/site-packages/distributed/worker.pyRi     s    c         C` s%   t  |  | d | d | d | d | S(   NR  R  R#  R  (   Ri   (   R  R   R  R  R#  R  (    (    s1   lib/python2.7/site-packages/distributed/worker.pyRj     s    c         c` s   | j  d t  } | } |  j | } t | |  } t |  d | }	 t |  rg | | |   V}
 nX | r |  j |	 t d | | | |  j |	 |  j	 |  j
 f d |  j V}
 n | | |   }
 t j i d d 6t |
  d 6  d  S(   Nt   separate_threadt   .R  R   RR  R   Rn  (   R   RX  R   t   getattrR:   R@   R3  t   apply_function_actorR  R   R   R   R   R   R&   (   R  R   R  R  R  R#  R4  R  R  R   Rn  (    (    s1   lib/python2.7/site-packages/distributed/worker.pyRw     s*    c         C` s.   t  |  j | |  } i d d 6t |  d 6S(   NRR  R   Rn  (   R6  R   R&   (   R  R   R  t	   attributeR  (    (    s1   lib/python2.7/site-packages/distributed/worker.pyRx     s    c         C` sR   | |  j  k r t Sx8 |  j  | j   D]# \ } } |  j | | k  r' t Sq' Wt S(   N(   R   RX  R:  R   R   (   R  R  R  t   needed(    (    s1   lib/python2.7/site-packages/distributed/worker.pyt   meets_resource_constraints  s     c         C` sU  |  j  r d  Sy x |  j r t |  j  |  j k  r |  j d } |  j j |  d k rl |  j j   q n  |  j |  r |  j j   |  j	 | d  q Pq Wxh |  j
 r
t |  j  |  j k  r
t j |  j
  \ } } |  j j |  t k r |  j	 | d  q q WWnB t k
 rP} t j |  t rJd d  l } | j   n    n Xd  S(   Ni    RJ   RI   (   R   RJ   R+  RI   R   R   R   R
  R:  R  RH   R  t   heappopR  Rz  R   R{  R  R  R  (   R  R  t   _R|  R  (    (    s1   lib/python2.7/site-packages/distributed/worker.pyRy    s,    	$$c         c` s  d  } |  j d' k r d  Szy8| |  j k s= | |  j k rA d  S|  j r{ | |  j k s_ t  |  j | d k s{ t  n  |  j | \ } } } t   } i  } xw |  j	 | D]h }	 y |  j
 |	 | |	 <Wq t k
 rd d l m }
 |
 t |  j |	  |  j |	 |   | |	 <q Xq Wt | | d t t f } t | | d t t f } t   } | | d k r|  j | j d | | f  |  j d  k	 r|  j d	 j | |  qn  t j d
 | |  j  yA |  j | t d | | | |  j | |  j |  j |  j f V} Wn t  k
 r(} | }   n X|  j j! |  d( k rEd  S| | d <| j" d d   } |  j | j d | d | d f  | d |  j# | <| d d k r| d |  j$ | <| d |  j% | <|  j& | d d | |  j d  k	 r|  j d j | d | d  qn t' | j" d  t(  rv|  j) j* i d d 6| d 6 |  j& | d d t+ |  j, | d t+ n | d |  j- | <| d |  j. | <t j/ d  t0 t1 |   d!  t2 | d" d! t3 | d" d! t4 | d j
   |  j& | d#  t j d$ | |  |  j r?| |  j k s't  | |  j k s?t  n  |  j5   |  j6   Wna t7 k
 r} | | k rt j8 d% |  qt j9 |  t: rd& d  l; } | j<   n    n XWd  | |  j k r|  j j= |  n  Xd  S()   NRF  RE  RI   i   (   R  t	   key_typesg{Gzt?s	   disk-reads   disk-load-durations   Execute key: %s worker: %sR  s   long-runningR  Rn  t   computeRc  RV  R  RH  s   task-finishedR   R@  RR   R  s   task-durations   actual-exceptiont
   rescheduleRT   Rz   R{  R   sH    Compute Failed
Function:  %s
args:      %s
kwargs:    %s
Exception: %s
i  t   max_lenRS   s*   Send compute response to scheduler: %s, %ss   Thread Pool Executor error: %si    (   RF  RE  (   RI   s   long-running(>   R   R   RI   R   RQ   R   R   R   R!   R   R   R  R  R  R@  R   RY   RB   t   bytesR   R   R  R  R  R   Rq  R3  t   apply_functionR  R   R   R   t   RuntimeErrorR   R   R   R   R   R  R   t
   RescheduleR   R  R   R  R   R   Rt  R  R.   t   convert_args_to_strt   convert_kwargs_to_strt   reprRy  Rx  Rz  RS   R{  R  R  R  R  (   R  R  Rz   t   executor_errorR  R  R#  Rc  R   R8  R  t   args2R  RV  Rn  R|  R  R  (    (    s1   lib/python2.7/site-packages/distributed/worker.pyR    s     		1	
%&	
c         c` sZ  |  j  r d St |  _  d } |  j j } | j   j } | |  j } |  j r | |  j k r |  j j	   |  j
 st j d t | d  t | j   j  t |  j   t |  _
 qnW |  j
 rt j d t | d  t | j   j  t |  j   t |  _
 |  j   n  |  j r>| |  j k r>|  j |  j } d } | | } x | | k r|  j j st j d t | j   j  t |  j   Pn  |  j j j   \ } }	 }
 ~ ~	 | |
 7} | d 7} t j V| j   j } | | k rK| | k rK|  j j	   | j   j } qKqKW| r>t j d | t |   q>n  t |  _  t j |   d S(	   s    Track this process's memory usage and act accordingly

        If we rise above 70% memory use, start dumping data to disk.

        If we rise above 80% memory use, stop execution of new tasks
        Ni    s^   Worker is at %d%% memory usage. Pausing worker.  Process memory: %s -- Worker memory limit: %sid   s^   Worker is at %d%% memory usage. Resuming worker. Process memory: %s -- Worker memory limit: %ss   Memory use is high but worker has no data to store to disk.  Perhaps some other process is leaking memory?  Process memory: %s -- Worker memory limit: %si   s+   Moved %d pieces of data data and %s to disk(   R  RX  R;  t   proct   memory_infot   rssR   R`   R  t   collectR   R   Rt  R   R;   R   Ry  R_   R^   R   t   fastt   evictR   t   momentRq  R   (   R  R  RJ  RR   t   fracR&  t   countt   needR8  RG  R   (    (    s1   lib/python2.7/site-packages/distributed/worker.pyR  R	  sf    					


	c         C` so   t    |  j } |  j t j   } |  _ |  j j | | f  |  j j | t |  j	  f  |  j	 j
   d  S(   N(   R!   R   R   R   R   R   R  R   R   R   R  (   R  RJ  t   prof(    (    s1   lib/python2.7/site-packages/distributed/worker.pyR  	  s
    c   
      ` sL  |  j  s d St   } |  j  |  j  j   } Wd QXt j       f d   | D   i  } |  j ry d   | D } n  x   j   D] \ } } | d k	 r t	 | |  } | j
 |  } t j | t |  j d d } t j | d |  t j | t |  j | d d q q Wt   }	 |  j d k	 rH|  j d j |	 |  n  d S(   sz   
        Get a frame from all actively computing threads

        Merge these frames into existing profile counts
        Nc         ` s   i  |  ] }   | |  q S(    (    (   R7  t   ident(   t   frames(    s1   lib/python2.7/site-packages/distributed/worker.pys
   <dictcomp>	  s   	 c         S` s"   i  |  ] } t  j |  |  q S(    (   R   t   ll_get_stack(   R7  RU  (    (    s1   lib/python2.7/site-packages/distributed/worker.pys
   <dictcomp>	  s   	 RV  s   distributed/worker.pys   profile-duration(   R   R!   R   R   R   t   _current_framesR  R:  R   R:   R   R   t   processRX  R   t	   llprocessR   R  R  (
   R  Rc  R   t   llframesRU  t   frameR  t   llframeR  RV  (    (   RV  s1   lib/python2.7/site-packages/distributed/worker.pyR  	  s,    		
	!	c         C` s  t    |  j } | d  k r( |  j } n8 g  |  j D]( \ } } | | k r2 | | | f ^ q2 } | d  k ru d }	 n t j | | f  }	 | d  k r d  }
 n4 t j | | f  d }
 |
 t |  k r d  }
 n  |	 d k r |
 d  k r t	 |  } nD |
 d  k rt |  n |
 } g  t
 |	 |  D] } | | ^ q(} t j t d |    } | sft j   S|
 d  k r| d  k s| | k  r| d  k r|  j } n |  j | } t j | |  } n  | S(   Ni    i   (   R!   R   R   R   R   t   bisectt   bisect_leftt   bisect_rightR+  R   R  R   R   R	   R   R   R   (   R  R   Rc  RV  R  RJ  t   historyt   tR  t   istartt   istopt   iistopR  RT  R<  (    (    s1   lib/python2.7/site-packages/distributed/worker.pyR  	  s2    8			&
$c   
      C` s6  | d  k r t } n  t   |  j } | p. | } | p: d } i g  |  j D]8 \ } } | | k  om | k  n rJ | | d f ^ qJ d 6g  |  j D]D \ } } | | k  o | k  n r | d   | j   D f ^ q d 6}	 | r2|	 d j | |  j d f  |	 d j | d   |  j	 j   D f  n  |	 S(   Ni    RR  t   countsc         S` s#   i  |  ] \ } } | d  |  q S(   RR  (    (   R7  R8  R  (    (    s1   lib/python2.7/site-packages/distributed/worker.pys
   <dictcomp>	  s   	 Ru   c         S` s#   i  |  ] \ } } | d  |  q S(   RR  (    (   R7  R8  RG  (    (    s1   lib/python2.7/site-packages/distributed/worker.pys
   <dictcomp>
  s   	 (
   R   RX  R!   R   R   R   R:  R  R   R   (
   R  R   Rc  RV  t
   add_recentRJ  Rb  R  RG  Rn  (    (    s1   lib/python2.7/site-packages/distributed/worker.pyR	  	  s    	IB
#c         ` s   |  j  < t j     |  j j   }   f d   | j   D   Wd  QX d  k	 rr  f d     j   D   n  d     j   D } | S(   Nc         ` s#   i  |  ] \ } }   | |  q S(    (    (   R7  RU  R8  (   RV  (    s1   lib/python2.7/site-packages/distributed/worker.pys
   <dictcomp>
  s   	 c         ` s+   i  |  ]! \ } } |   k r | |  q S(    (    (   R7  R8  R\  (   Ru   (    s1   lib/python2.7/site-packages/distributed/worker.pys
   <dictcomp>
  s   	 c         S` s(   i  |  ] \ } } t  j |  |  q S(    (   R   Rr   (   R7  R8  R\  (    (    s1   lib/python2.7/site-packages/distributed/worker.pys
   <dictcomp>
  s   	 (   R   R   RX  R   R   R:  R   (   R  R   Ru   R   Rn  (    (   RV  Ru   s1   lib/python2.7/site-packages/distributed/worker.pyR  
  s    
"c         C` s   | |  j  k s$ | |  j k s$ t  | |  j k s9 t  | |  j k sN t  | |  j k sc t  | |  j k sx t  | |  j k r |  j | d k s t  n  d  S(   NRR   (   R   R   R   R   R   RI   RH   R   (   R  R  (    (    s1   lib/python2.7/site-packages/distributed/worker.pyt   validate_key_memory
  s    $c         ` sl   |   j  k s t  |   j k s* t  |   j k s? t  t   f d     j | D  sh t  d  S(   Nc         3` s-   |  ]# } |   j  k p$ |   j k Vq d  S(   N(   R   R   (   R7  R  (   R  (    s1   lib/python2.7/site-packages/distributed/worker.pys	   <genexpr>$
  s    (   RI   R   R   R   R  R   (   R  R  (    (   R  s1   lib/python2.7/site-packages/distributed/worker.pyt   validate_key_executing
  s
    c         ` s   | t  d   j  k s t  |   j k s3 t  |   j k sH t  |   j k s] t  t   f d     j | D  s t  d  S(   Ni   c         3` s-   |  ]# } |   j  k p$ |   j k Vq d  S(   N(   R   R   (   R7  R  (   R  (    s1   lib/python2.7/site-packages/distributed/worker.pys	   <genexpr>-
  s    (   R	   RH   R   R   RI   R   R  R   (   R  R  (    (   R  s1   lib/python2.7/site-packages/distributed/worker.pyt   validate_key_ready'
  s    c         ` sC   |   j  k s t  t   f d     j | D  s? t  d  S(   Nc         3` s   |  ] } |   j  k Vq d  S(   N(   R   (   R7  R  (   R  (    s1   lib/python2.7/site-packages/distributed/worker.pys	   <genexpr>2
  s    (   R   R   R  R   (   R  R  (    (   R  s1   lib/python2.7/site-packages/distributed/worker.pyt   validate_key_waiting0
  s    c         C` s   y |  j  | } | d k r, |  j |  nT | d k rH |  j |  n8 | d k rd |  j |  n | d k r |  j |  n  WnB t k
 r } t j |  t r d d  l	 } | j
   n    n Xd  S(   NRR   RG   RH   RI   i    (   R   Rh  Rk  Rj  Ri  Rz  R   R{  R  R  R  (   R  R  R  R|  R  (    (    s1   lib/python2.7/site-packages/distributed/worker.pyR  4
  s     c         ` sk   |   j  k s t  |   j k s* t    j | s= t  t   f d     j | D  sg t  d  S(   Nc         3` s   |  ] } |   j  k Vq d  S(   N(   RH   (   R7  R  (   R  (    s1   lib/python2.7/site-packages/distributed/worker.pys	   <genexpr>K
  s    (   R   R   R   R   R  (   R  R  (    (   R  s1   lib/python2.7/site-packages/distributed/worker.pyt   validate_dep_waitingG
  s    c         ` s~   |   j  k s t  |   j k s* t  t   f d     j | D  sT t    j | } |   j | k sz t  d  S(   Nc         3` s   |  ] } |   j  k Vq d  S(   N(   RH   (   R7  R  (   R  (    s1   lib/python2.7/site-packages/distributed/worker.pys	   <genexpr>P
  s    (   R   R   R   R  R   R   R   (   R  R  t   peer(    (   R  s1   lib/python2.7/site-packages/distributed/worker.pyt   validate_dep_flightM
  s
    *c         C` s}   | |  j  k s$ | |  j k s$ t  | |  j k s9 t  | |  j k sN t  | |  j k ry |  j | d k sy t  n  d  S(   NRR   (   R   R   R   R   R   R   (   R  R  (    (    s1   lib/python2.7/site-packages/distributed/worker.pyt   validate_dep_memoryT
  s
    $c         C` s   yt |  j  | } | d k r, |  j |  nG | d k rH |  j |  n+ | d k rd |  j |  n t d |   WnB t k
 r } t j |  t r d d  l	 } | j
   n    n Xd  S(   NRG   RU   RR   s   Unknown dependent statei    (   R   Rl  Rn  Ro  Ra  Rz  R   R{  R  R  R  (   R  R  R  R|  R  (    (    s1   lib/python2.7/site-packages/distributed/worker.pyR  [
  s    c         C` s5  |  j  d k r d  SyxG |  j j   D]6 \ } } x' | D] } | |  j | k s9 t  q9 Wq& WxG |  j j   D]6 \ } } x' | D] } | |  j | k s t  q Wqp Wx |  j D] } |  j |  q Wx |  j D] } |  j |  q Wx} |  j	 j   D]l \ } } | |  j
 k r xN | D]C } | |  j k s| |  j k s|  j | j |  j  st  qWq q Wx| |  j D]q } |  j | d k rvt |  j | t  st  | |  j	 k st  | |  j k s| |  j k st  qvqvWWnB t k
 r0}	 t j |	  t r*d d  l }
 |
 j   n    n Xd  S(   NRQ  RR   i    (   R   R   R:  R   R   R   R  R   R  R   R   R   R   t   issubsetR   R   R   R   R   R   R   Rz  R   R{  R  R  R  (   R  R  R  R  R]   Ru   R8  R  R  R|  R  (    (    s1   lib/python2.7/site-packages/distributed/worker.pyR   n
  s<    !!*/c         C` s.   |  j   |  j r |  j S|  j   SWd  QXd  S(   N(   R   R   t   _get_client(   R  (    (    s1   lib/python2.7/site-packages/distributed/worker.pyt   client
  s    
	i   c         C` s  y d d l  m } |   } Wn t k
 r0 nC X| j rR | j j |  j j k sg | j |  j j k rs | |  _ n  |  j sd d l  m } |  j t	 j
   k } | |  j d |  j d |  j d t d | d t d	 d
 d | |  _ | s|  j j d k st  qn  |  j S(   s    Get local client attached to this worker

        If no such client exists, create one

        See Also
        --------
        get_client
        i   (   t   default_client(   t   ClientR   R   t   set_as_defaultt   asynchronoust   direct_to_workersR   R]   RD  RQ  (   Rr  Rs  Ra  R   RY   t
   _start_argR   Rt  R   R   R   R   RX  R   R   (   R  RD  Rs  Rr  Rt  Rv  (    (    s1   lib/python2.7/site-packages/distributed/worker.pyRq  
  s0    						c         C` s   |  j  t   S(   s   Get the key of the task we are currently running

        This only makes sense to run within a task

        Examples
        --------
        >>> from dask.distributed import get_worker
        >>> def f():
        ...     return get_worker().get_current_task()

        >>> future = client.submit(f)  # doctest: +SKIP
        >>> future.result()  # doctest: +SKIP
        'f-1234'

        See Also
        --------
        get_worker
        (   R   R   (   R  (    (    s1   lib/python2.7/site-packages/distributed/worker.pyt   get_current_task
  s    (c   R*  t
   __module__t   __doc__R   RX  R   R   R   R  R,  R   t   propertyR4  R?  RB  R   R  Ro  R   Re  Rq   Rp   Ru   Rh   Rt   R  Rf  R  Rc  R  Ry   R  R  Rn   R  R  Rk   Rl   Rm   R  R   R
  R  R   R   R   R   R  R   R   R   R   R   R  R   R   R  R  R  Rx  R  R  R  R  R%  R  R#  R'  R  R  R  R/  R3  Ri   Rj   Rw   Rx   R:  Ry  R  R  R  R  R  R	  R  Rh  Ri  Rj  Rk  R  Rl  Rn  Ro  R  R   Rr  Rq  Ry  (    (    (    s1   lib/python2.7/site-packages/distributed/worker.pyRK   c   s   4		
	P		E		<				Nn	%				2			^	+	u		8			D)	!			qO		!&												-)c          C` sd   y t  j d SWnN t k
 r_ x. t d d d  D] }  |    } | r3 | Sq3 Wt d   n Xd S(   s   Get the worker currently running this task

    Examples
    --------
    >>> def f():
    ...     worker = get_worker()  # The worker on which this task is running
    ...     return worker.address

    >>> future = client.submit(f)  # doctest: +SKIP
    >>> future.result()  # doctest: +SKIP
    'tcp://127.0.0.1:47373'

    See Also
    --------
    get_client
    worker_client
    R]   Nis   No workers found(   R8   R  t   AttributeErrorR  Ra  (   R  R]   (    (    s1   lib/python2.7/site-packages/distributed/worker.pyt
   get_worker
  s    	i   c         C` s   |  r | r t  j |   }  n  y t   } Wn t k
 r> n* X|  sX | j j |  k rh | j d |  Sd d l m } |   } | r |  s | j j |  k r | S|  r d d l m	 } | |  d | St d   d S(   s  Get a client while within a task.

    This client connects to the same scheduler to which the worker is connected

    Parameters
    ----------
    address : str, optional
        The address of the scheduler to connect to. Defaults to the scheduler
        the worker is connected to.
    timeout : int, default 3
        Timeout (in seconds) for getting the Client
    resolve_address : bool, default True
        Whether to resolve `address` to its canonical form.

    Returns
    -------
    Client

    Examples
    --------
    >>> def f():
    ...     client = get_client()
    ...     futures = client.map(lambda x: x + 1, range(10))  # spawn many tasks
    ...     results = client.gather(futures)
    ...     return sum(results)

    >>> future = client.submit(f)  # doctest: +SKIP
    >>> future.result()  # doctest: +SKIP
    55

    See Also
    --------
    get_worker
    worker_client
    secede
    RD  i   (   t   _get_global_client(   Rt  s.   No global client found and no address providedN(
   R   t   resolve_addressR~  Ra  R   RY   Rq  Rr  R  Rt  (   RY   RD  R  R]   R  Rr  Rt  (    (    s1   lib/python2.7/site-packages/distributed/worker.pyt
   get_client
  s     %	c          C` sC   t    }  t   t   t j } |  j j |  j t j d | d S(   s  
    Have this task secede from the worker's thread pool

    This opens up a new scheduling slot and a new thread for a new task. This
    enables the client to schedule tasks on this node, which is
    especially useful while waiting for other jobs to finish (e.g., with
    ``client.gather``).

    Examples
    --------
    >>> def mytask(x):
    ...     # do some work
    ...     client = get_client()
    ...     futures = client.map(...)  # do some remote work
    ...     secede()  # while that work happens, remove ourself from the pool
    ...     return client.gather(futures)  # return gathered results

    See Also
    --------
    get_client
    get_worker
    R  N(	   R~  t
   tpe_secedeR!   R8   t
   start_timeR   Rd  R  R  (   R]   R  (    (    s1   lib/python2.7/site-packages/distributed/worker.pyR-   :  s    			RD  c           B` s   e  Z d  Z RS(   s   Reschedule this task

    Raising this exception will stop the current execution of the task and ask
    the scheduler to reschedule this task, possibly on a different machine.

    This does not guarantee that the task will move onto a different machine.
    The scheduler will proceed through its normal heuristics to determine the
    optimal machine to accept this task.  The machine will likely change if the
    load across the cluster has significantly changed since first scheduling
    the task.
    (   R*  Rz  R{  (    (    (    s1   lib/python2.7/site-packages/distributed/worker.pyRD  [  s   c         C` s   |  d  k r d  S|  d k r< t t t d | |   }  n  t t t  : t |   } t | t  r | d k r t | t  SWd  QXt |  t	 t
 f  r t |   St |   Sd  S(   NRL   i   (   R   R   t   TOTAL_MEMORYR  R4   Ra  t	   TypeErrorR   R   R   R  R>   (   R   R   t   total_corest   x(    (    s1   lib/python2.7/site-packages/distributed/worker.pyR   k  s     
c   
      c` s   | d k r |  j } n  | d k r0 |  j } n  |  j |  V} d | _ z t | d | d | d d d | d | d | V} y | d	 }	 Wn  t k
 r t d
 |   n X|	 d k r | j d  Vn  Wd |  j	 | |  Xt
 j |   d S(   s   Get keys from worker

    The worker has a two step handshake to acknowledge when data has been fully
    delivered.  This function implements that handshake.

    See Also
    --------
    Worker.get_data
    Worker.gather_deps
    utils_comm.gather_data_from_workers
    s#   Ephemeral Worker->Worker for gatherRM  RO  RH  Rk   Ru   R  R  R   s   Unexpected responseRR  N(   R   RM  RO  R   R   R   R  Ra  RZ  t   reuseR   R   (
   R  Ru   R]   R  R  RM  RO  R   Ri  R   (    (    s1   lib/python2.7/site-packages/distributed/worker.pyR!  {  s.    	
c         C` s   |  d k	 r t j |   }  n  | r6 t j |  } n  | rN t j |  } n  | d k	 r |  ro | ro | su t  t }  | f } n  |  | p d | p i  f S(   s>    Deserialize task inputs and regularize to func, args, kwargs N(    (   R   R%   Rb  R   t   execute_task(   R  R  R#  R  (    (    s1   lib/python2.7/site-packages/distributed/worker.pyR    s    c         C` s^   t  |   r4 |  d |  d } } | t t |    St |  t  rV t t t |    S|  Sd S(   s    Evaluate a nested task

    >>> inc = lambda x: x + 1
    >>> execute_task((inc, 1))
    2
    >>> execute_task((sum, [1, 2, (inc, 3)]))
    7
    i    i   N(   R   t   mapR  R   R   (   R  R  R  (    (    s1   lib/python2.7/site-packages/distributed/worker.pyR    s    	(   t   LRUi R   c         C` s
   t  |  S(   N(   R+  (   R8  RG  (    (    s1   lib/python2.7/site-packages/distributed/worker.pyRN     RO   c         C` ss   y t  |  } Wn^ t k
 rO t j |   } t |  d k  ro | t  |  <qo n  t k
 rn t j |   } n X| S(   s+    Dump a function to bytes, cache functions i (   t   cacheR  R%   R  R+  R  (   R  Rn  (    (    s1   lib/python2.7/site-packages/distributed/worker.pyR    s    c         C` s   t  |   r |  d t k r t t t |  d   r i t |  d  d 6t |  d  d 6} t |   d k r t |  d  | d <n  | St t t |  d   s i t |  d  d 6t |  d  d 6Sn  t |   S(	   s>   Serialize a dask task

    Returns a dict of bytestrings that can each be loaded with ``loads``

    Examples
    --------
    Either returns a task as a function, args, kwargs dict

    >>> from operator import add
    >>> dumps_task((add, 1))  # doctest: +SKIP
    {'function': b' 	_operatoradd.'
     'args': b'   KK.'}

    Or as a single task blob if it can't easily decompose the result.  This
    happens either if the task is highly nested, or if it isn't a task at all

    >>> dumps_task(1)  # doctest: +SKIP
    {'task': b'       K.'}
    i    i   i   R  R  i   i   R#  (	   R   R   R  R  R2   R  t
   warn_dumpsR+  R&   (   R  R  (    (    s1   lib/python2.7/site-packages/distributed/worker.pyt
   dumps_task  s    *()g    .Ac         C` s   | |   } t  d r t |  | k r t t  d <t |   } t |  d k rj | d  d | d } n  t j d t t |   | f  n  | S(   s8    Dump an object to bytes, warn if those bytes are large i    iF   i2   s    ... isN  Large object of size %s detected in task graph: 
  %s
Consider scattering large objects ahead of time
with client.scatter to reduce scheduler burden and 
keep data on workers

    future = client.submit(func, big_data)    # bad

    big_future = client.scatter(big_data)     # good
    future = client.submit(func, big_future)  # good(   t   _warn_dumps_warnedR+  RX  R  R  R  R;   (   t   objR  t   limitt   bt   s(    (    s1   lib/python2.7/site-packages/distributed/worker.pyR    s    
c         C` s&  t    } |  | | | <Wd QXt   t _ | t _ | t _ t   }	 z y |  | |   }
 Wn3 t k
 r } t |  } d | d <| | d <nH Xi d d 6d d 6|
 d 6t |
  d	 6|
 d k	 r t
 |
  n d d
 6} Wd t   } X|	 | | d <| | | d <| | d <|  | | =Wd QX| S(   s    Run a function, collect information

    Returns
    -------
    msg: dictionary with status, result/error, timings, etc..
    Ns
   task-erredRH  s   actual-exceptions   task-finishedRR  R   Rn  R   R@  Rc  RV  R  (   R   R!   R8   R  R  R  Rz  R   R  R   R@  (   R  R  R#  R  R  R   R   t
   time_delayRU  Rc  Rn  R|  R  Ru  (    (    s1   lib/python2.7/site-packages/distributed/worker.pyRB  "  s6    				 
&

c   	      C` sY   t    } |  | | | <Wd QX| t _ | t _ |  | |   } |  | | =Wd QX| S(   s    Run a function, collect information

    Returns
    -------
    msg: dictionary with status, result/error, timings, etc..
    N(   R   R8   R  R  (	   R  R  R#  R  R  R   R   RU  Rn  (    (    s1   lib/python2.7/site-packages/distributed/worker.pyR7  Q  s    				c         C` sr   d t  f d     Y} |  j   }  d |  k rH | t |  d  |  d <n  d |  k rn | t |  d  |  d <n  |  S(   s    Make a worker msg, which contains args and kwargs, safe to cast to str:
    allowing for some arguments to raise exceptions during conversion and
    ignoring them.
    t   Reprc           B` s   e  Z d    Z d   Z RS(   c         S` s   | |  _  | |  _ d  S(   N(   t   _ft   _val(   R  R  t   val(    (    s1   lib/python2.7/site-packages/distributed/worker.pyR  q  s    	c         S` s   |  j  |  j  S(   N(   R  R  (   R  (    (    s1   lib/python2.7/site-packages/distributed/worker.pyR,  u  s    (   R*  Rz  R  R,  (    (    (    s1   lib/python2.7/site-packages/distributed/worker.pyR  p  s   	R  R#  (   t   objectR   RE  RF  (   R  R  (    (    s1   lib/python2.7/site-packages/distributed/worker.pyt   get_msg_safe_strj  s    c         C` s   d } g  t  t |    D] } d ^ q } x t |   D] \ } } y t |  } Wn t k
 rm d } n X| | | <| t |  d 7} | d	 k	 r8 | | k r8 d j d j | | d    |  Sq8 Wd j d j |   Sd	 S(
   sx    Convert args to a string, allowing for some arguments to raise
    exceptions during conversion and ignoring them.
    i    RO   s    < could not convert arg to str >i   s   ({}s   , i   s   ({})N(   R  R+  t	   enumerateRG  Rz  R   R  R   (   R  R@  t   lengthR  t   strst   argt   sarg(    (    s1   lib/python2.7/site-packages/distributed/worker.pyRE    s    %

&c   	      C` s  d } g  t  t |    D] } d ^ q } x t |  j    D] \ } \ } } y t |  } Wn t k
 ry d } n Xt |  d | } | | | <| t |  d 7} | d
 k	 r> | | k r> d j d j | | d    |  Sq> Wd	 j d j |   Sd
 S(   sz    Convert kwargs to a string, allowing for some arguments to raise
    exceptions during conversion and ignoring them.
    i    RO   s    < could not convert arg to str >s   : i   s   {{{}s   , i   s   {{{}}}N(	   R  R+  R  R:  RG  Rz  R   R  R   (	   R#  R@  R  R  R  t   argnameR  R  t   skwarg(    (    s1   lib/python2.7/site-packages/distributed/worker.pyRF    s    %%

&c         C` s
   t  |  S(   N(   R  (   R8  RG  (    (    s1   lib/python2.7/site-packages/distributed/worker.pyR     s    c   
      c` s  t  j |  } | d  k r* t |  } n t j d  | sO | sO t d   | rg t  j |  } n  | r t  j |  } n  t | d  r |  | d <n  t | d  r |  | d <n  t j	 d t
 |   yQ | s | | |   } n5 | r| | |   V} n |  j j | | |  d  } Wnc t k
 r} t j d t t
 |   d  t | d d t | d d d	 t t |  }	 n Xi d
 d 6t |  d 6}	 t |	   d  S(   Ns\   The is_coro= parameter is deprecated. We now automatically detect coroutines/async functionss   Combination not supportedRS  t   dask_schedulers   Run out-of-band function %rs3    Run Failed
Function: %s
args:     %s
kwargs:   %s
i  R@  R  RR  R   Rn  (   R%   Rb  R   R@   R  R  R   R1   R   RW  R.   R   Rd  Rz  Rt  R  RE  RF  RX  R   R&   R   (
   t   serverR   R  R  R#  t   is_coroR  Rn  R|  Ri  (    (    s1   lib/python2.7/site-packages/distributed/worker.pyRi     s@    
(   t
   __future__R    R   R   R^  t   collectionsR   R   t   datetimeR   R  R0  R   R%   R   R  R   R   R  R  R   t	   dask.coreR   t   dask.compatibilityR   t   cytoolzR	   R
   R   R   t   toolzt   tornado.genR   t   tornadoR   t   tornado.ioloopR   t   tornado.locksR   RO   R   R   t   batchedR   R   R   R   t
   comm.utilsR   t   compatibilityR   R   R   R   R=  R   R   R   R   R   t	   diskutilsR    R  R!   t   nodeR"   t
   preloadingR#   t	   proctitleR$   R  R&   R'   R(   t   pubsubR)   R   R*   R  R+   t   threadpoolexecutorR,   R-   R  t   utilsR.   R/   R0   R1   R2   R3   R4   R5   R6   R7   R8   R9   R:   R;   R<   R=   R>   R?   R@   RA   t
   utils_commRB   RC   t
   utils_perfRD   RE   RF   t	   cpu_countR   t	   getLoggerR*  R   R   R   R  R  t   psutilt   virtual_memoryR  R  Rt  R   R  R.  R,  R  R  R  RK   R~  RX  R  Rz  RD  R   R  R!  R1  R  R  R   R  R   R  R  R  R   R  R  R  RB  R7  R  RE  RF  R   Ri   (    (    (    s1   lib/python2.7/site-packages/distributed/worker.pyt   <module>   s     "("
	          	<	!+						/			