U
    f6                     @   s  d dl mZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ d d	lZ	d d	l
Z
d d	lZd d	lZd d	lZd d	lZd d	lZd d
lmZ d d	lZd d	lZd d	lZd d	lZzd dlmZ W n  ek
r   d dlmZ Y nX d dlmZmZ dddddddddddddddddddd d!gZdddddddddd"d#d$ddddddd%d&d'gZd(d)d*d+d,d-d.d/d0d1d2d3gZd4d5d6d7d8d9gZd4d5d6d7d8d9gZd:d;d<d7d=d8d>d9gZ d:d;d<d7d=d8d>d9gZ!d?d@d7d8d9gZ"d?d@d7d8d9gZ#g Z$e$dAdB eD 7 Z$e$dCdB eD 7 Z$e$dDdB eD 7 Z$e$dEdB eD 7 Z$e$dFdB eD 7 Z$e$dGdB eD 7 Z$e$dHdB eD 7 Z$e$dIdB eD 7 Z$e$dJdB eD 7 Z$e$dKdB eD 7 Z$e$dLdB eD 7 Z$e$dMdB eD 7 Z$e$dNdB eD 7 Z$e$dOdB eD 7 Z$e$dPdB eD 7 Z$e$dQdB eD 7 Z$e$dRdB eD 7 Z$e$dSdB eD 7 Z$e$dTdB eD 7 Z$dkdUdVZ%dldYdZZ&d[d\ Z'dmd]d^Z(d a)d a*d a+d a,i a-e Z.d a/d_d` Z0dadb Z1dndddeZ2dodgdhZ3didj Z4d	S )p    )print_function   )datasets   )metrics)models)methods)__version__N)Pool)Queue)ThreadLockZruntimeZlocal_accuracyZconsistency_guaranteesZkeep_positive_maskZkeep_positive_resampleZkeep_positive_imputeZkeep_negative_maskZkeep_negative_resampleZkeep_negative_imputeZkeep_absolute_mask__r2Zkeep_absolute_resample__r2Zkeep_absolute_impute__r2Zremove_positive_maskZremove_positive_resampleZremove_positive_imputeZremove_negative_maskZremove_negative_resampleZremove_negative_imputeZremove_absolute_mask__r2Zremove_absolute_resample__r2Zremove_absolute_impute__r2Zkeep_absolute_mask__roc_aucZkeep_absolute_resample__roc_aucZkeep_absolute_impute__roc_aucZremove_absolute_mask__roc_aucZ!remove_absolute_resample__roc_aucZremove_absolute_impute__roc_aucZhuman_and_00Zhuman_and_01Zhuman_and_11Zhuman_or_00Zhuman_or_01Zhuman_or_11Zhuman_xor_00Zhuman_xor_01Zhuman_xor_11Zhuman_sum_00Zhuman_sum_01Zhuman_sum_11Zlinear_shap_corrZlinear_shap_indZcoefrandomZkernel_shap_1000_meanrefZsampling_shap_1000Ztree_shap_tree_path_dependentZtree_shap_independent_200ZsaabasZ	tree_gainZmean_abs_tree_shapZ	deep_shapZexpected_gradientsc                 C   s"   g | ]}t D ]}d d||gqqS )corrgroups60lassolinear_regress_methods.0sm r   A/tmp/pip-target-lpfmz8o1/lib/python/shap/benchmark/experiments.py
<listcomp>   s       r   c                 C   s"   g | ]}t D ]}d d||gqqS )r   ridger   r   r   r   r   r      s       c                 C   s"   g | ]}t D ]}d d||gqqS )r   decision_treetree_regress_methodsr   r   r   r   r      s       c                 C   s"   g | ]}t D ]}d d||gqqS )r   random_forestr   r   r   r   r   r      s       c                 C   s"   g | ]}t D ]}d d||gqqS )r   gbmr   r   r   r   r   r      s       c                 C   s"   g | ]}t D ]}d d||gqqS )r   ffnndeep_regress_methodsr   r   r   r   r      s       c                 C   s"   g | ]}t D ]}d d||gqqS )independentlinear60r   r   r   r   r   r   r      s       c                 C   s"   g | ]}t D ]}d d||gqqS )r#   r   r   r   r   r   r   r      s       c                 C   s"   g | ]}t D ]}d d||gqqS )r#   r   r   r   r   r   r   r      s       c                 C   s"   g | ]}t D ]}d d||gqqS )r#   r   r   r   r   r   r   r      s       c                 C   s"   g | ]}t D ]}d d||gqqS )r#   r   r   r   r   r   r   r      s       c                 C   s"   g | ]}t D ]}d d||gqqS )r#   r    r!   r   r   r   r   r      s       c                 C   s"   g | ]}t D ]}d d||gqqS )cricr   linear_classify_methodsr   r   r   r   r      s       c                 C   s"   g | ]}t D ]}d d||gqqS )r$   r   r%   r   r   r   r   r      s       c                 C   s"   g | ]}t D ]}d d||gqqS )r$   r   tree_classify_methodsr   r   r   r   r      s       c                 C   s"   g | ]}t D ]}d d||gqqS )r$   r   r'   r   r   r   r   r      s       c                 C   s"   g | ]}t D ]}d d||gqqS )r$   r   r'   r   r   r   r   r      s       c                 C   s"   g | ]}t D ]}d d||gqqS )r$   r    )deep_classify_methodsr   r   r   r   r      s       c                 C   s"   g | ]}t D ]}d d||gqqS )Zhumanr   r   r   r   r   r   r      s       c                 c   sl   t D ]b}| d k	r| |d krq|d k	r4||d kr4q|d k	rJ||d krJq|d k	r`||d kr`q|V  qd S )Nr   r   r      )_experiments)datasetmodelmethodmetric
experimentr   r   r   experiments   s    r1   T/tmpc              
   C   s   | \}}}}t | }tj||d }|r^tj|r^t|d}	t|	W  5 Q R  S Q R X t|	dddd  t
j  t }
tt| \}}tt|||tt|d | |}tdt |
   t|d}	t||	 W 5 Q R X |S )	N.picklerb__    z ...z...took %f seconds.
wb)__gen_cache_idospathjoinisfileopenpickleloadprintreplacesysstdoutflushtimegetattrr   r   r   dump)r0   Z	use_cache	cache_dirdataset_name
model_namemethod_namemetric_namecache_id
cache_filefstartXyZscorer   r   r   run_experiment   s(    
 rT   c                 C   s   | \}}t ||dS )N)rI   )rT   )argsr0   rI   r   r   r   run_experiments_helper   s    rV   c           	   	   C   sr   t t| |||d}|dkr8t ttt|t|}n,t|}|tt|t|}W 5 Q R X t t||S )N)r,   r-   r.   r/   r   )listr1   maprV   zip	itertoolsrepeatr
   )	r,   r-   r.   r/   rI   ZnworkersZexperiments_arroutpoolr   r   r   run_experiments   s    
"r^   c                 C   sB  | d\}}d}t  zH|tkr,g t|< tt| tk rBd}nt t| t   dkr`d}W 5 t  X |rtqtd q| 	 }d}t
j|t|d }t
j|st  zt| t  W 5 t  X t  ztd7 at  W 5 t  X t||||d	 t  ztd7 at  W 5 t  X |   qd S )
N:FT=      r2   r3   r   )rI   python_binary)splitworker_lockacquirereleasehost_recordslenssh_conn_per_min_limitrF   sleepgetr:   r;   r<   r9   r=   append
total_sent__print_status__run_remote_experiment
total_done	task_done)qhosthostnamerb   Z	all_clearr0   rI   rO   r   r   r   __thread_worker   sD    





ru   c                   C   s*   t dttttt f dd tj  d S )Nz4Benchmark task %d of %d done (%d failed, %d running))end)rA   rp   nexperimentstotal_failedrm   rC   rD   rE   r   r   r   r   rn   =  s    rn   
   c           	   	   C   s   |a t|}t| t|D ]R}|d\}}ztjd|dgdd W q  tjk
rp   t	d|d Y  dS X q tt
| } t|  t| ad	ad	ad	ai at }|D ]&}tt||fd
}|d |  q| D ]}|| q|  dS )a   Use ssh to run the experiments on remote machines in parallel.

    Parameters
    ----------
    experiments : iterable
        Output of shap.benchmark.experiments(...).

    thread_hosts : list of strings
        Each host has the format "host_name:path_to_python_binary" and can appear multiple times
        in the list (one for each parallel execution you want on that machine).

    rate_limit : int
        How many ssh connections we make per minute to each host (to avoid throttling issues).
    r_   sshz&pkill -f shap.benchmark.run_experiment   )timeoutzFailed to connect tozafter 15 seconds! Exiting.Nr   )targetrU   T)ri   copyr   shufflesetrc   
subprocessrunTimeoutExpiredrA   rW   rh   rx   rm   rp   ry   rg   r   r   ru   	setDaemonrQ   putr<   )	r1   Zthread_hostsZ
rate_limitrs   rt   _rr   Zworkerr0   r   r   r   run_remote_experimentsB  s2    




r   pythonc              
   C   sX  | \}}}}t | }tj||d }	tj|	rZt|	d}
t|
W  5 Q R  S Q R X t	t
dd d| d|||||||f  }ztd||g W n\ tjk
 r } z:td| tjd	 t|tjd	 td
7 at| W Y d S d }~X Y nX td|d |	 |	g tj|	rLt|	d}
t|
W  5 Q R  S Q R X ntdd S )Nr3   r4   r   ra   zCUDA_VISIBLE_DEVICES="" zj -c "import shap; shap.benchmark.run_experiment(['%s', '%s', '%s', '%s'], cache_dir='%s')" &> %s/%s.outputr{   z#The following command failed on %s:)filer   Zscpr_   z;Remote benchmark call finished but no local file was found!)r9   r:   r;   r<   r=   r>   r?   r@   rF   rj   r   uniformr   check_outputCalledProcessErrorrA   rC   stderrry   	Exception)r0   remoterI   rb   rJ   rK   rL   rM   rN   rO   rP   cmder   r   r   ro   v  s:          ro   c                 C   s$   | \}}}}dd t||||g S )Nvr5   )r<   r	   )r0   rJ   rK   rL   rM   r   r   r   r9     s    r9   )NNNN)Tr2   )NNNNr2   r   )rz   )r2   r   )5
__future__r    r   r   r   r   r	   numpynpZsklearnr:   r?   rC   rF   r   multiprocessingr
   rZ   r   r   queuer   ImportError	threadingr   r   Zregression_metricsZbinary_classification_metricsZhuman_metricsr   r&   r   r(   r"   r)   r+   r1   rT   rV   r^   rx   rm   rp   ry   rg   rd   ri   ru   rn   r   ro   r9   r   r   r   r   <module>   s>  
	



=
4
&