σ
ίόΪ\c           @` s,  d  d l  m Z m Z m Z d  d l Z d  d l Z d  d l Z d  d l Z d  d l Z d  d l	 Z	 y d  d l
 m Z Wn! e k
 r d  d l m Z n Xd  d l m Z d  d l m Z d  d l m Z e j e  Z d d d     YZ d	   Z d d
  Z d d d  Z d e f d     YZ d S(   i    (   t   print_functiont   divisiont   absolute_importN(   t   Queue(   t   Thread(   t   merge(   t   gent   bcolorsc           B` s8   e  Z d  Z d Z d Z d Z d Z d Z d Z d Z	 RS(   s   [95ms   [94ms   [92ms   [93ms   [91ms   [0ms   [1ms   [4m(
   t   __name__t
   __module__t   HEADERt   OKBLUEt   OKGREENt   WARNINGt   FAILt   ENDCt   BOLDt	   UNDERLINE(    (    (    s5   lib/python2.7/site-packages/distributed/deploy/ssh.pyR      s   c   
      ` s
  d d  l  } d d l m   d d l m } m } | j   } | j | j    d } xqt	 rΚyb t
 j d  j t
 j  | j d  d d  d d	  d
 d  d d t	 d d d d  PWqZ | | f k
 rΖ} t d t j d j d  d d	  d
 d  d  t j  t t j d t |  t j  t j   | d 7} | d k rt d t j d t j  t j d  n  t d t j d j d | d d  t j  t j d  qZ XqZ Wt d j d  d d  d   | j d  d d  d! t	 \ }    j   j d"      f d#        f d$        f d%   } x.  d& j   r‘t j d'  |   rtPqtqtWt j   }	 xA t j   |	 d( k  rρ j d)  |   rαPn  t j d'  q±W j    | j    d  S(*   Ni    (   t   PipeTimeout(   t   SSHExceptiont   PasswordRequiredExceptiont   paramikot   hostnamet   addresst   usernamet   ssh_usernamet   portt   ssh_portt   key_filenamet   ssh_private_keyt   compresst   timeouti   t   banner_timeouts   [ dask-ssh ] : sC   SSH connection error when connecting to {addr}:{port}to run '{cmd}'t   addrt   cmds,                  SSH reported this exception: i   i   s/   SSH connection failed after 3 retries. Exiting.s                  s!   Retrying... (attempt {n}/{total})t   nt   totals   [ {label} ] : {cmd}t   labels   $SHELL -i -c 't   't   get_ptygΉ?c          ` s   yx  j    }  xe t |   d k rv |  j   }  t j d |    d j d j d  d d |     j    }  q WWn   t j f k
 r n Xd S(   s<   
        Read stdout stream, time out if necessary.
        i    s   stdout from ssh channel: %st   output_queues   [ {label} ] : {output}R%   t   outputN(	   t   readlinet   lent   rstript   loggert   debugt   putt   formatt   socketR   (   t   line(   R   t   cmd_dictt   stdout(    s5   lib/python2.7/site-packages/distributed/deploy/ssh.pyt   read_from_stdout   s    
	c          ` s±   y  j    }  x} t |   d k r |  j   }  t j d |    d j d j d  d  t j d j d |   t j	   j    }  q WWn   t
 j f k
 r¬ n Xd S(	   s<   
        Read stderr stream, time out if necessary.
        i    s   stderr from ssh channel: %sR(   s   [ {label} ] : R%   s   {output}R)   N(   R*   R+   R,   R-   R.   R/   R0   R   R   R   R1   R   (   R2   (   R   R3   t   stderr(    s5   lib/python2.7/site-packages/distributed/deploy/ssh.pyt   read_from_stderr   s    
5c          ` sk           j    rg   j   }   d j d j d  d  t j d t |   t j  t Sd S(   sp   
        Communicate a little bit, without blocking too long.
        Return True if the command ended.
        R(   s   [ {label} ] : R%   s'   remote process exited with exit status N(	   t   exit_status_readyt   recv_exit_statusR/   R0   R   R   t   strR   t   True(   t   exit_status(   t   channelR3   R7   R5   (    s5   lib/python2.7/site-packages/distributed/deploy/ssh.pyt   communicate¦   s    
3t   input_queueg      π?g      @s   (!   R   t   paramiko.buffered_pipeR   t   paramiko.ssh_exceptionR   R   t	   SSHClientt   set_missing_host_key_policyt   AutoAddPolicyR;   t   loggingt	   getLoggert   setLevelt   WARNt   connectt   printR   R   R0   R   R:   t	   tracebackt	   print_exct   ost   _exitt   timet   sleept   exec_commandR=   t
   settimeoutt   emptyt   sendt   close(
   R3   R   R   R   t   ssht   retriest   et   stdinR>   t   start(    (   R   R=   R3   R7   R5   R6   R4   s5   lib/python2.7/site-packages/distributed/deploy/ssh.pyt	   async_ssh&   sh    		



	=

+	$$			
c         C` s*  d j  d | p t j d | d |   } |  d  k	 rn d j  d |   | } | d j  d | d | d |   7} n  t j d j  d | d |  t j } t   }	 t   }
 i	 | d	 6| d
 6| d 6| d 6|	 d 6|
 d 6| d 6| d 6| d 6} t d t	 d | g  } t
 | _ | j   t | i | d 6 S(   Ns8   {python} -m distributed.cli.dask_scheduler --port {port}t   pythonR   t   logdirs   mkdir -p {logdir} && s,   &> {logdir}/dask_scheduler_{addr}:{port}.logR!   s   scheduler {addr}:{port}R"   R%   R   R?   R(   R   R   R   t   targett   argst   thread(   R0   t   syst
   executablet   NoneR   R   R   R   R   R[   R;   t   daemonRZ   R   (   R]   R!   R   R   R   R   t   remote_pythonR"   R%   R?   R(   R3   R`   (    (    s5   lib/python2.7/site-packages/distributed/deploy/ssh.pyt   start_schedulerΠ   s.    	&		
	
s   distributed.cli.dask_workerc         C` s  d | d k r d n d } |	 s/ | d 7} n  |
 rB | d 7} n  | rU | d 7} n  | rh | d 7} n  | j  d	 | p} t j d
 | d | d | d | d | d | d |
 d | d |  
} |  d  k	 rϊ d j  d |   | } | d j  d | d |   7} n  d j  d |  } t   } t   } i | d 6| d 6| d 6| d 6| d 6| d 6| d 6| d 6} t d  t d! | g  } t | _ | j	   t
 | i | d" 6 S(#   NsX   {python} -m {remote_dask_worker} {scheduler_addr}:{scheduler_port} --nthreads {nthreads}i   s    --nprocs {nprocs}t    s    --host {worker_addr}s    --memory-limit {memory_limit}s    --worker-port {worker_port}s    --nanny-port {nanny_port}R\   t   remote_dask_workert   scheduler_addrt   scheduler_portt   worker_addrt   nthreadst   nprocst   memory_limitt   worker_portt
   nanny_ports   mkdir -p {logdir} && R]   s%   &> {logdir}/dask_scheduler_{addr}.logR!   s   worker {addr}R"   R%   R   R?   R(   R   R   R   R^   R_   R`   (   R0   Ra   Rb   Rc   R   R   R[   R;   Rd   RZ   R   (   R]   Ri   Rj   Rk   Rl   Rm   R   R   R   t   nohostRn   Ro   Rp   Re   Rh   R"   R%   R?   R(   R3   R`   (    (    s5   lib/python2.7/site-packages/distributed/deploy/ssh.pyt   start_workerώ   sR    				
	
t
   SSHClusterc           B` s   e  Z d  d d d d e d d d d d d d  Z e j d    Z e d    Z	 d   Z
 d   Z d	   Z d
   Z d   Z RS(   i    i   i   s   distributed.cli.dask_workerc         C` sI  | |  _  | |  _ | |  _ | |  _ | |  _ | |  _ | |  _ |	 |  _ | |  _ | |  _	 | |  _
 | |  _ | |  _ d d  l } |
 d  k	 rί t j j |
 d | j j   j d   }
 t t j d j d |
  t j  n  |
 |  _ g  |  _ t |
 | | | | | |  |  _ g  |  _ x' t |  D] \ } } |  j |  q(Wd  S(   Ni    s	   dask-ssh_s   %Y-%m-%d_%H:%M:%Ssa   Output will be redirected to logfiles stored locally on individual worker nodes under "{logdir}".R]   (    Ri   Rj   Rl   Rm   R   R   R   Rq   Re   Rn   Ro   Rp   Rh   t   datetimeRc   RM   t   patht   joint   nowt   strftimeRJ   R   R   R0   R   R]   t   threadsRf   t	   schedulert   workerst	   enumeratet
   add_worker(   t   selfRi   Rj   t   worker_addrsRl   Rm   R   R   R   Rq   R]   Re   Rn   Ro   Rp   Rh   Rt   t   iR!   (    (    s5   lib/python2.7/site-packages/distributed/deploy/ssh.pyt   __init__P  sB    														$			c         C` s   d  S(   N(    (   R~   (    (    s5   lib/python2.7/site-packages/distributed/deploy/ssh.pyt   _start  s    c         C` s   d |  j  |  j f S(   Ns   %s:%d(   Ri   Rj   (   R~   (    (    s5   lib/python2.7/site-packages/distributed/deploy/ssh.pyt   scheduler_address  s    c         C` s   |  j  g |  j } yZ xS t rk x9 | D]1 } x( | d j   sV t | d j    q/ Wq& Wt j d  q WWn t k
 r n Xd  S(   NR(   gΉ?(	   Rz   R{   R;   RS   RJ   t   getRO   RP   t   KeyboardInterrupt(   R~   t   all_processest   process(    (    s5   lib/python2.7/site-packages/distributed/deploy/ssh.pyt   monitor_remote_processes‘  s    	c         C` sn   |  j  j t |  j |  j |  j | |  j |  j |  j |  j	 |  j
 |  j |  j |  j |  j |  j |  j   d  S(   N(   R{   t   appendRr   R]   Ri   Rj   Rl   Rm   R   R   R   Rq   Rn   Ro   Rp   Re   Rh   (   R~   R   (    (    s5   lib/python2.7/site-packages/distributed/deploy/ssh.pyR}   ΅  s"    	c         C` sG   |  j  g |  j } x- | D]% } | d j d  | d j   q Wd  S(   NR?   t   shutdownR`   (   Rz   R{   R/   Rv   (   R~   R   R   (    (    s5   lib/python2.7/site-packages/distributed/deploy/ssh.pyR   Κ  s    c         C` s   |  S(   N(    (   R~   (    (    s5   lib/python2.7/site-packages/distributed/deploy/ssh.pyt	   __enter__Ρ  s    c         G` s   |  j    d  S(   N(   R   (   R~   R_   (    (    s5   lib/python2.7/site-packages/distributed/deploy/ssh.pyt   __exit__Τ  s    N(   R   R	   Rc   t   FalseR   R   t	   coroutineR   t   propertyR   R   R}   R   R   R   (    (    (    s5   lib/python2.7/site-packages/distributed/deploy/ssh.pyRs   O  s&   9				(    (   t
   __future__R    R   R   RE   R1   RM   Ra   RO   RK   t   queueR   t   ImportErrort	   threadingR   t   toolzR   t   tornadoR   RF   R   R-   R   R[   Rc   Rf   Rr   t   objectRs   (    (    (    s5   lib/python2.7/site-packages/distributed/deploy/ssh.pyt   <module>   s(   	«;B