ó
ßüÚ\c           @` sÈ  d  d l  m Z m Z m Z d  d l m Z d  d l Z d  d l Z d  d l m	 Z	 d  d l
 Z
 d  d l m Z m Z d  d l m Z d  d l m Z d  d l m Z m Z m Z d  d	 l m Z d  d
 l m Z d  d l m Z d  d l m Z m Z d  d l m Z m Z m  Z  m! Z! m" Z" m# Z# m$ Z$ m% Z% d  d l m& Z& d „  Z' e d e( d d d d ƒ d „  ƒ Z) e d e( d d ƒ d „  ƒ Z* d „  Z+ e d e( d e d d5 g d d i d d 6ƒ d  „  ƒ Z, e d e d d d e( ƒ d! „  ƒ Z- e d e d e( d d ƒ d" „  ƒ Z. e d e d e( d d ƒ d# „  ƒ Z/ d$ „  Z0 d% „  Z1 e d e d e( d d ƒ d& „  ƒ Z2 d' „  Z3 d( „  Z4 e d e d e( d d ƒ d) „  ƒ Z5 e d e d d ƒ d* „  ƒ Z6 e d e d d ƒ d+ „  ƒ Z7 e d e d e( d d ƒ d, „  ƒ Z8 e d e( d d d d ƒ d- „  ƒ Z9 e d e( d e d d ƒ d. „  ƒ Z: e d e( d d ƒ d/ „  ƒ Z; e! e d e( d d d e d d6 g ƒ d0 „  ƒ ƒ Z< e d e( d1 i d2 d3 6ƒ d4 „  ƒ Z= d S(7   i    (   t   print_functiont   divisiont   absolute_import(   t   CancelledErrorN(   t   sleep(   t   partition_allt   first(   t   gen(   t   delayed(   t   Clientt   Nannyt   wait(   t   CommClosedError(   R   (   t   time(   t   synct   ignoring(   t   gen_clustert   clustert   inct   slowt   divt   slowinct   slowaddt   captured_logger(   t   loopc         C` s¹   t  d d ƒ ¤ \ } \ } } t | d d |  ƒy } | j t t d ƒ ƒ } t | ƒ | d ƒ  j ƒ  | j t | ƒ } | j	 ƒ  t t t t d ƒ ƒ ƒ k s© t
 ‚ Wd  QXWd  QXd  S(   Nt   active_rpc_timeouti
   t   addressR   t   proc(   R   R	   t   mapR   t   rangeR   t	   terminatet   submitt   sumt   resultt   AssertionError(   R   t   st   at   bt   ct   Lt   total(    (    sD   lib/python2.7/site-packages/distributed/tests/test_failed_workers.pyt$   test_submit_after_failed_worker_sync   s    
t   clientt   timeouti<   R   i
   c         c` sà   t  | j | j d d d | j ƒ} | j d ƒ x' t | j ƒ d k  rZ t j d ƒ Vq4 W|  j	 t
 t d ƒ ƒ } t | ƒ V| j j | j ƒ |  j t | ƒ } | V} | t t	 t
 t d ƒ ƒ ƒ k sÑ t ‚ | j ƒ  Vd  S(   Nt   ncoresi   R   i    i   gš™™™™™¹?i
   (   R
   t   ipt   portR   t   startt   lent   workersR   R   R   R   R   R   t   add_callbackt   killR   R    R"   t   close(   R&   R#   R$   R%   t   nR'   R(   R!   (    (    sD   lib/python2.7/site-packages/distributed/tests/test_failed_workers.pyt%   test_submit_after_failed_worker_async)   s    $'c         c` sr   |  j  t t d ƒ ƒ } t | ƒ V| j ƒ  V|  j t | ƒ } | V} | t t  t t d ƒ ƒ ƒ k sn t ‚ d  S(   Ni
   (   R   R   R   R   R4   R   R    R"   (   R&   R#   R$   R%   R'   R(   R!   (    (    sD   lib/python2.7/site-packages/distributed/tests/test_failed_workers.pyt   test_submit_after_failed_worker;   s    c         C` s°   t  d d ƒ › \ } \ } } t | d d |  ƒp } | j t t d ƒ ƒ } t | ƒ | d ƒ  j ƒ  | j | ƒ } | t t t t d ƒ ƒ ƒ k s  t	 ‚ Wd  QXWd  QXd  S(   NR   i
   R   R   R   (
   R   R	   R   R   R   R   R   t   gathert   listR"   (   R   R#   R$   R%   R&   R'   R!   (    (    sD   lib/python2.7/site-packages/distributed/tests/test_failed_workers.pyt   test_gather_after_failed_workerF   s    
t   WorkerR,   s	   127.0.0.1i   i   t   configt   1ss!   distributed.comm.timeouts.connectc         c` s  |  j  t t d ƒ ƒ } t | ƒ V| j j j j ƒ  |  j t | ƒ } x¸ t d ƒ D]ª } t | ƒ Vt	 | j
 | j j ƒ j }	 x= | | | g D], }
 |
 j |	 k r’ |
 j j j j ƒ  Pq’ q’ W|  j | g ƒ V} | t t  t t d ƒ ƒ ƒ g k sU t ‚ qU Wd  S(   Ni   i   (   R   R   R   R   t   processt   _processR   R   R    R   t   taskst   keyt   who_hasR   t   worker_addressR8   R"   (   R&   R#   t   wt   xt   yt   zR'   R(   t   it   addrt   workerR!   (    (    sD   lib/python2.7/site-packages/distributed/tests/test_failed_workers.pyt,   test_gather_then_submit_after_failed_workersP   s    c   	   	   c` sÛ  |  j  t t d ƒ ƒ } t | ƒ V| j } t t ƒ % |  j t j	 d d | j
 g ƒVWd  QXt ƒ  } x: | j | k r¤ t j d ƒ Vt ƒ  | d k  sk t ‚ qk Wt j d ƒ Vt ƒ  } x@ t | j ƒ d k  rþ t j d ƒ Vt ƒ  | d k  s¿ t ‚ q¿ Wt | ƒ V|  j  t t d d ƒ ƒ } t | ƒ Vt d „  | j j ƒ  Dƒ ƒ sUt ‚ t | j ƒ } |  j ƒ  V|  j  t t d ƒ ƒ } t | ƒ Vt d	 „  | j j ƒ  Dƒ ƒ s·t ‚ t | ƒ t | j ƒ @s×t ‚ d  S(
   Ni
   i   R1   g{®Gáz„?g      à?i   i   c         s` s!   |  ] } t  | ƒ d  k Vq d S(   i    N(   R0   (   t   .0t   keys(    (    sD   lib/python2.7/site-packages/distributed/tests/test_failed_workers.pys	   <genexpr>   s    c         s` s!   |  ] } t  | ƒ d  k Vq d S(   i    N(   R0   (   RL   RM   (    (    sD   lib/python2.7/site-packages/distributed/tests/test_failed_workers.pys	   <genexpr>ˆ   s    (   R   R   R   R   t   pidR   R   t   _runt   ost   _exitRC   R   R   R   R"   R0   R,   t   allt   has_whatt   valuest   dictt   _restartt   set(	   R&   R#   R$   R%   R'   t   original_pidR/   t   L2t   ncores2(    (    sD   lib/python2.7/site-packages/distributed/tests/test_failed_workers.pyt"   test_failed_worker_without_warningi   s0    	&		%%c         c` s†  | j  i d | j 6d | j 6k s) t ‚ |  j t d ƒ } |  j t | ƒ } |  j t d d ƒ } | Vt | j ƒ | j | j h k sŽ t ‚ |  j	 ƒ  V} | |  k s­ t ‚ t
 | j ƒ d k sÈ t ‚ t d „  | j j ƒ  Dƒ ƒ sî t ‚ | j sþ t ‚ | j ƒ  st ‚ | j ƒ  s"t ‚ | j ƒ  s4t ‚ | j | j k sLt ‚ | j s\t ‚ t d „  | j j ƒ  Dƒ ƒ s‚t ‚ d  S(   Ni   i   i    c         s` s   |  ] } | j  Vq d  S(   N(   t	   occupancy(   RL   t   ws(    (    sD   lib/python2.7/site-packages/distributed/tests/test_failed_workers.pys	   <genexpr>œ   s    c         s` s   |  ] } | j  Vq d  S(   N(   t
   wants_what(   RL   t   cs(    (    sD   lib/python2.7/site-packages/distributed/tests/test_failed_workers.pys	   <genexpr>¦   s    (   R,   RC   R"   R   R   R   RW   RB   RA   RV   R0   R1   t   anyRT   t	   cancelledt
   exceptionst	   who_wantst   clients(   R&   R#   R$   R%   RE   RF   RG   t   f(    (    sD   lib/python2.7/site-packages/distributed/tests/test_failed_workers.pyt   test_restart   s"    )'&c         c` sj   d t  d ƒ d } |  j | ƒ } t | g ƒ V|  j ƒ  Vx' | j | j g D] } | sO t ‚ qO Wd  S(   Ni   i   (   R   t   computeR   RV   R@   t
   unrunnableR"   (   R&   R#   R$   R%   RE   Re   t   coll(    (    sD   lib/python2.7/site-packages/distributed/tests/test_failed_workers.pyt   test_restart_cleared©   s    c         C` s½   t  d t ƒ ¨ \ } \ } } t | d d |  ƒ} } | j t d ƒ } | j ƒ  | j ƒ  se t ‚ | j t d ƒ } | j ƒ  d k s t ‚ t	 | j
 ƒ  ƒ d k s­ t ‚ Wd  QXWd  QXd  S(   Nt   nannyR   R   i   i   i   (   R   t   TrueR	   R   R   t   restartRa   R"   R!   R0   R,   (   R   R#   R$   R%   R&   RE   RF   (    (    sD   lib/python2.7/site-packages/distributed/tests/test_failed_workers.pyt   test_restart_sync_no_centerµ   s    
c         C` s$  t  d t ƒ \ } \ } } t | d d |  ƒä } | j t d d ƒ } | j ƒ  t |  | j j ƒ sq t	 ‚ | j
 ƒ  t |  | j j ƒ s— t	 ‚ | j ƒ  s© t	 ‚ t | j ƒ  ƒ d k sÇ t	 ‚ t j t ƒ  | j ƒ  Wd  QX| j t d d ƒ } | j ƒ  d k st	 ‚ Wd  QXWd  QXd  S(   NRk   R   R   i   i   i   gUUUUUUÕ?(   R   Rl   R	   R   R   R!   R   t	   schedulerRB   R"   Rm   Ra   R0   R,   t   pytestt   raisesR   (   R   R#   R$   R%   R&   RE   RF   (    (    sD   lib/python2.7/site-packages/distributed/tests/test_failed_workers.pyt   test_restart_syncÀ   s    

c         c` s«   |  j  t t d ƒ ƒ } t ƒ  } |  j ƒ  Vt ƒ  | d k  sE t ‚ t | j ƒ d k s` t ‚ t d „  | Dƒ ƒ s| t ‚ |  j	 t
 d ƒ } | V} | d k s§ t ‚ d  S(   Ni
   i   c         s` s   |  ] } | j  d  k Vq d S(   Ra   N(   t   status(   RL   RE   (    (    sD   lib/python2.7/site-packages/distributed/tests/test_failed_workers.pys	   <genexpr>Ü   s    i   (   R   R   R   R   RV   R"   R0   R,   RR   R   R   (   R&   R#   R$   R%   R'   R/   RE   R!   (    (    sD   lib/python2.7/site-packages/distributed/tests/test_failed_workers.pyt   test_restart_fastÓ   s    	c         C` s˜   t  d t d d ƒ } \ } \ } t | d d |  ƒU } | j t d ƒ } t d ƒ t ƒ  } | j ƒ  t ƒ  } | | d k  sˆ t ‚ Wd  QXWd  QXd  S(	   NRk   t   nworkersi   R   R   id   gš™™™™™¹?i   (   R   Rl   R	   R   R   R   Rm   R"   (   R   R#   RD   R&   t   futureR/   t   stop(    (    sD   lib/python2.7/site-packages/distributed/tests/test_failed_workers.pyt(   test_worker_doesnt_await_task_completionã   s    !
	
	c         C` sï   t  d t ƒ Ú \ } \ } } t | d d |  ƒ¯ } | j t t d ƒ ƒ } t ƒ  } | j ƒ  t ƒ  | d k  s{ t ‚ t	 | j
 ƒ  ƒ d k s™ t ‚ t d „  | Dƒ ƒ sµ t ‚ | j t d ƒ } | j ƒ  d k sß t ‚ Wd  QXWd  QXd  S(   NRk   R   R   i
   i   c         s` s   |  ] } | j  d  k Vq d S(   Ra   N(   Rs   (   RL   RE   (    (    sD   lib/python2.7/site-packages/distributed/tests/test_failed_workers.pys	   <genexpr>ø   s    i   (   R   Rl   R	   R   R   R   R   Rm   R"   R0   R,   RR   R   R   R!   (   R   R#   R$   R%   R&   R'   R/   RE   (    (    sD   lib/python2.7/site-packages/distributed/tests/test_failed_workers.pyt   test_restart_fast_syncî   s    	
c         c` s   |  j  t t d ƒ ƒ } t ƒ  } |  j ƒ  Vt ƒ  | d k  sE t ‚ t d „  | Dƒ ƒ sa t ‚ |  j t d ƒ } | V} | d k sŒ t ‚ d  S(   Ni
   c         s` s   |  ] } | j  d  k Vq d S(   Ra   N(   Rs   (   RL   RE   (    (    sD   lib/python2.7/site-packages/distributed/tests/test_failed_workers.pys	   <genexpr>  s    i   i   (	   R   R   R   R   RV   R"   RR   R   R   (   R&   R#   R$   R%   R'   R/   RE   R!   (    (    sD   lib/python2.7/site-packages/distributed/tests/test_failed_workers.pyt   test_fast_killþ   s    	c   	      c` sé   t  |  j |  j f d t ƒV} t  |  j |  j f d t ƒV} | j t d ƒ } | j t d ƒ } | V} | V} | d k s‚ t ‚ | d k s” t ‚ | j ƒ  V| j ƒ  s± t ‚ | j ƒ  sÃ t ‚ | j	 d t ƒ V| j	 d t ƒ Vd  S(   Nt   asynchronousi   i   i   t   fast(
   R	   R-   R.   Rl   R   R   R"   RV   Ra   t   _close(	   R#   R$   R%   t   e1t   e2RE   RF   t   xxt   yy(    (    sD   lib/python2.7/site-packages/distributed/tests/test_failed_workers.pyt   test_multiple_clients_restart  s    c         c` sv   d d  l  } | j ƒ  | j | j f } |  j ƒ  Vt |  j ƒ d k sN t ‚ | j | j f } | | k sr t ‚ d  S(   Ni    i   (   t   gct   collectRC   Rm   R0   R,   R"   (   R#   R$   R%   Rƒ   t   addrst   addrs2(    (    sD   lib/python2.7/site-packages/distributed/tests/test_failed_workers.pyt   test_restart_scheduler"  s    
c         c` s_   |  j  t d ƒ } |  j ƒ  V|  j  t d ƒ } ~ d d  l } | j ƒ  t j d ƒ V| Vd  S(   Ni   i    gš™™™™™¹?(   R   R   RV   Rƒ   R„   R   R   (   R&   R#   R$   R%   RE   RF   Rƒ   (    (    sD   lib/python2.7/site-packages/distributed/tests/test_failed_workers.pyt0   test_forgotten_futures_dont_clean_up_new_futures/  s    
c      
   c` sh  d | _  t | j | j d d d | j ƒ} | j d ƒ t ƒ  } x@ t | j ƒ d k  r… t	 j
 d ƒ Vt ƒ  | d k  sF t ‚ qF Wd	 } | | d
 d } d } |  j t t | ƒ d g  t | ƒ D] }	 d | |	 f ^ qÆ ƒ}
 xt t |
 ƒ d
 k r[| d
 7} |  j t d g  t t |
 ƒ d ƒ D] }	 d | |	 f ^ q't t d |
 ƒ Œ  Œ}
 qè Wt	 j
 t j ƒ  d ƒ Vt t ƒ % |  j t j d
 d | j g ƒVWd  QXt	 j
 t j ƒ  d ƒ Vx' t | j ƒ d k  rèt	 j
 d ƒ VqÂWt t t ƒ % |  j t j d
 d | j g ƒVWd  QX|  j |
 ƒ V\ } t | t ƒ sGt ‚ | | k sYt ‚ | j ƒ  Vd  S(   Nid   R,   i   R   i    i   g{®Gáz„?i   i   i   RA   s	   inc-%d-%ds	   add-%d-%di   R1   (   t   allowed_failuresR
   R-   R.   R   R/   R   R0   R,   R   R   R"   R   R   R   R   t   zipR   t   randomR   R   RO   RP   RQ   RC   R1   t   EnvironmentErrorR8   t
   isinstancet   intR4   (   R&   R#   R$   R%   R5   R/   t   Nt   expected_resultRH   t   jR'   R!   (    (    sD   lib/python2.7/site-packages/distributed/tests/test_failed_workers.pyt%   test_broken_worker_during_computation<  s:    	$	A
L&&c         c` s!  g  t  d ƒ D] } t t ƒ | d d ƒ^ q } g  | D] } t t ƒ | d d ƒ^ q8 } g  t | | ƒ D]' \ } } t t ƒ | | d d ƒ^ ql }	 t t ƒ |	 ƒ }
 |  j |
 ƒ } t j d ƒ V| j	 s× t
 ‚ |  j ƒ  V| j	 sò t
 ‚ t | j ƒ d k st
 ‚ | j st
 ‚ d  S(   Ni2   t   delayg{®Gáz„?g      à?i   (   R   R   R   RŠ   R   R    Rg   R   R   t   rprocessingR"   RV   R0   R,   R@   (   R&   R#   R$   R%   RH   t   xst   ysRE   RF   t   zsR(   R!   (    (    sD   lib/python2.7/site-packages/distributed/tests/test_failed_workers.pyt   test_restart_during_computationg  s    1+=c      
   #` sö  t  | j | j d d d | j ƒ} | j d ƒ t ƒ  } x@ t | j ƒ d k  r| t j	 d ƒ Vt ƒ  | d k  s= t
 ‚ q= W|  j t t d ƒ d	 d d
 g  t d ƒ D] } d | ^ q¥ ƒ} t | ƒ V|  j t | d | j ƒV} x7 t | j ƒ t | j ƒ D] }	 | j |	 d t ƒqW| j ‰  t t ƒ " |  j t j d d ˆ  g ƒVWd  QXx' t | j ƒ d k rt j	 d ƒ Vq[W|  j t | d | j ƒ}
 |
 V| j j ˆ  ƒ s»t
 ‚ t  ‡  f d †  | j! j" ƒ  Dƒ ƒ sçt
 ‚ | j# ƒ  Vd  S(   NR,   i   R   i    i   g{®Gáz„?i   i   R“   RA   s   f%dR1   t   reporti   c         3` s   |  ] } ˆ  | k Vq d  S(   N(    (   RL   R#   (   t   n_worker_address(    sD   lib/python2.7/site-packages/distributed/tests/test_failed_workers.pys	   <genexpr>”  s    ($   R
   R-   R.   R   R/   R   R0   R,   R   R   R"   R   R   R   R   R   R    R   RW   t	   dep_statet
   task_statet   release_depRl   RC   R   R   RO   RP   RQ   R1   RS   t   getR`   RB   RT   R4   (   R&   R#   R$   R%   R5   R/   RH   t   futuresR!   t   depR(   (    (   Rš   sD   lib/python2.7/site-packages/distributed/tests/test_failed_workers.pyt2   test_worker_who_has_clears_after_failed_connectionx  s*    $	A#	#,c         c` sn   t  d ƒ 8 } |  j t d ƒ } t j d ƒ V|  j d d ƒ VWd  QX| j ƒ  } d | j ƒ  k sj t ‚ d  S(   Ns   distributed.scheduleri  gš™™™™™¹?R+   i   (   R   R   R   R   Rm   t   getvaluet   lowerR"   (   R&   R#   R$   t   sioRv   t   text(    (    sD   lib/python2.7/site-packages/distributed/tests/test_failed_workers.pyt)   test_restart_timeout_on_long_running_task™  s    t   scheduler_kwargst   100mst
   worker_ttlc         c` s  | j  d j ƒ  t j d ƒ Vt | j ƒ | j | j h k sF t ‚ t ƒ  } xL t | j ƒ | j | j h k r t j d ƒ Vt ƒ  | d k  sR t ‚ qR Wt | j ƒ | j h k t ƒ  } x: | j	 d k rü t j d ƒ Vt ƒ  | d k  sÃ t ‚ qÃ W| j	 d k st ‚ d  S(	   Nt	   heartbeatg{®Gáz„?gš™™™™™©?i   t   runningt   closedt   closing(   R¬   R­   (
   t   periodic_callbacksRw   R   R   RW   R1   R   R"   R   Rs   (   R&   R#   R$   R%   R/   (    (    sD   lib/python2.7/site-packages/distributed/tests/test_failed_workers.pyt   test_worker_time_to_live¥  s    '	$	(   s	   127.0.0.1i   (   s	   127.0.0.1i   (>   t
   __future__R    R   R   t   concurrent.futuresR   RP   R‹   R   R   Rp   t   toolzR   R   t   tornadoR   t   daskR   t   distributedR	   R
   R   t   distributed.commR   t   distributed.clientt   distributed.metricst   distributed.utilsR   R   t   distributed.utils_testR   R   R   R   R   R   R   R   R   R)   Rl   R6   R7   R:   RK   R[   Rf   Rj   Rn   Rr   Rt   Rx   Ry   Rz   R‚   R‡   Rˆ   R’   R˜   R¡   R¦   R¯   (    (    (    sD   lib/python2.7/site-packages/distributed/tests/test_failed_workers.pyt   <module>   sT   :
	
$	
$$$$		$		$$$+$!0