U
    f{                     @   s   d dl mZ d dl mZ d dl mZ d dlmZ d dl mZ ddl mZ ddl mZ d	d
l	Z	d	d
l
Zd	d
lZd	d
lZd	d
lZd	d
lZd	d
lZzd	dlmZ W n  ek
r   d	dlmZ Y nX dd Zdd Zdd Zdd ZdddZdddZdddZdddZdddZddd Zdd!d"Zdd#d$Z dd%d&Z!dd'd(Z"dd)d*Z#dd+d,Z$dd-d.Z%dd/d0Z&dd1d2Z'dd3d4Z(dd5d6Z)dd7d8Z*dd9d:Z+dd;d<Z,dd=d>Z-dd?d@Z.ddAdBZ/ddCdDZ0ddEdFZ1ddGdHZ2ddIdJZ3ddKdLZ4dMdN Z5ddOdPZ6ddQdRZ7ddSdTZ8ddUdVZ9dWdX Z:i Z;dd\d]Z<d
a=d
a>d
a?d
a@d
aAd
aBd^d_ ZCd`da ZDdbdc ZEddde ZFdfdg ZGdhdi ZHdjdk ZIdldm ZJdndo ZKdpdq ZLdrds ZMdtdu ZNdvdw ZOdxdy ZPdzd{ ZQd|d} ZRd~d ZSdd ZTdd ZUdd ZVdd ZWd
S )   )LinearExplainer)KernelExplainer)SamplingExplainer)other)__version__   )measures)methods    N)train_test_splitc                 C   s   t j }t jd g }tdD ]}tt| |d|d\}}}	}
| }|||	 t }tt	|||}t | }t }|| t | }|
||d |jd    q"t j| dt |fS )z9 Runtime
    transform = "negate"
    sort_order = 1
      r   d   	test_sizerandom_stateg     @@r
   N)nprandomseedranger   	__toarrayfittimegetattrr	   appendshapemean)Xymodel_generatormethod_nameold_seedmethod_repsiX_trainX_testy_train_modelstartZ	explainerZ
build_timeZexplain_time r)   =/tmp/pip-target-lpfmz8o1/lib/python/shap/benchmark/metrics.pyruntime   s     
r+   c                    s,   dd  fdd}dt | |d ||fS )zB Local Accuracy
    transform = "identity"
    sort_order = 2
    c                 S   s   t dt||  t| d  }|dk r.dS |dk r:dS |dk rFdS |dk rRd	S |d
k r^dS |dk rjdS |dk rvd
S |dk rdS dS dS )z` Converts local accuracy from % of standard deviation to numerical scores for coloring.
              ?g:0yE>gư>g{Gz?g?g?g      ?g?333333?g?g?g333333?g      ?gffffff?        N)minr   Zstd)truepredvr)   r)   r*   	score_map>   s$    "z!local_accuracy.<locals>.score_mapc              
      s   t | ||||| |S N)r   local_accuracy)r#   r$   r%   y_testattr_functiontrained_modelr   r   r3   r)   r*   score_functionU   s          z&local_accuracy.<locals>.score_functionN)__score_method)r   r   r   r   r:   r)   r9   r*   r5   8   s    r5   c                 C   s.   ddddddddddddddd}d|| fS )zJ Consistency Guarantees
    transform = "identity"
    sort_order = 3
    r,   r.   g?r-   )Zlinear_shap_corrZlinear_shap_indZcoefZkernel_shap_1000_meanrefZsampling_shap_1000r   ZsaabasZ	tree_gainZtree_shap_tree_path_dependentZtree_shap_independent_200Zmean_abs_tree_shapZlime_tabular_regression_1000Z	deep_shapZexpected_gradientsNr)   )r   r   r   r   Z
guaranteesr)   r)   r*   consistency_guarantees\   s     r<   c                 C   s
   t |S )z? A trivial metric that is just is the output of the model.
    )r   r   )r0   r1   r)   r)   r*   __mean_predy   s    r=      c              	   C   s   t tj| |||d|tS )z Keep Positive (mask)
    xlabel = "Max fraction of features kept"
    ylabel = "Mean model output"
    transform = "identity"
    sort_order = 4
    r   __run_measurer   	keep_maskr=   r   r   r   r   num_fcountsr)   r)   r*   keep_positive_mask~   s    rD   c              	   C   s   t tj| |||d|tS )z Keep Negative (mask)
    xlabel = "Max fraction of features kept"
    ylabel = "Negative mean model output"
    transform = "negate"
    sort_order = 5
    r?   rB   r)   r)   r*   keep_negative_mask   s    rF   c              	   C   s   t tj| |||d|tjjS )z Keep Absolute (mask)
    xlabel = "Max fraction of features kept"
    ylabel = "R^2"
    transform = "identity"
    sort_order = 6
    r
   )r@   r   rA   sklearnmetricsr2_scorerB   r)   r)   r*   keep_absolute_mask__r2   s    rJ   c              	   C   s   t tj| |||d|tjjS )z Keep Absolute (mask)
    xlabel = "Max fraction of features kept"
    ylabel = "ROC AUC"
    transform = "identity"
    sort_order = 6
    r
   r@   r   rA   rG   rH   roc_auc_scorerB   r)   r)   r*   keep_absolute_mask__roc_auc   s    rM   c              	   C   s   t tj| |||d|tS )z Remove Positive (mask)
    xlabel = "Max fraction of features removed"
    ylabel = "Negative mean model output"
    transform = "negate"
    sort_order = 7
    r   r@   r   remove_maskr=   rB   r)   r)   r*   remove_positive_mask   s    rP   c              	   C   s   t tj| |||d|tS )z Remove Negative (mask)
    xlabel = "Max fraction of features removed"
    ylabel = "Mean model output"
    transform = "identity"
    sort_order = 8
    rE   rN   rB   r)   r)   r*   remove_negative_mask   s    rQ   c              	   C   s   t tj| |||d|tjjS )z Remove Absolute (mask)
    xlabel = "Max fraction of features removed"
    ylabel = "1 - R^2"
    transform = "one_minus"
    sort_order = 9
    r
   )r@   r   rO   rG   rH   rI   rB   r)   r)   r*   remove_absolute_mask__r2   s    rR   c              	   C   s   t tj| |||d|tjjS )z Remove Absolute (mask)
    xlabel = "Max fraction of features removed"
    ylabel = "1 - ROC AUC"
    transform = "one_minus"
    sort_order = 9
    r
   r@   r   rO   rG   rH   rL   rB   r)   r)   r*   remove_absolute_mask__roc_auc   s    rT   c              	   C   s   t tj| |||d|tS )z Keep Positive (resample)
    xlabel = "Max fraction of features kept"
    ylabel = "Mean model output"
    transform = "identity"
    sort_order = 10
    r   r@   r   keep_resampler=   rB   r)   r)   r*   keep_positive_resample   s    rW   c              	   C   s   t tj| |||d|tS )z Keep Negative (resample)
    xlabel = "Max fraction of features kept"
    ylabel = "Negative mean model output"
    transform = "negate"
    sort_order = 11
    rE   rU   rB   r)   r)   r*   keep_negative_resample   s    rX   c              	   C   s   t tj| |||d|tjjS )z Keep Absolute (resample)
    xlabel = "Max fraction of features kept"
    ylabel = "R^2"
    transform = "identity"
    sort_order = 12
    r
   )r@   r   rV   rG   rH   rI   rB   r)   r)   r*   keep_absolute_resample__r2   s    rY   c              	   C   s   t tj| |||d|tjjS )z Keep Absolute (resample)
    xlabel = "Max fraction of features kept"
    ylabel = "ROC AUC"
    transform = "identity"
    sort_order = 12
    r
   )r@   r   rV   rG   rH   rL   rB   r)   r)   r*   keep_absolute_resample__roc_auc   s    rZ   c              	   C   s   t tj| |||d|tS )z Remove Positive (resample)
    xlabel = "Max fraction of features removed"
    ylabel = "Negative mean model output"
    transform = "negate"
    sort_order = 13
    r   r@   r   remove_resampler=   rB   r)   r)   r*   remove_positive_resample   s    r]   c              	   C   s   t tj| |||d|tS )z Remove Negative (resample)
    xlabel = "Max fraction of features removed"
    ylabel = "Mean model output"
    transform = "identity"
    sort_order = 14
    rE   r[   rB   r)   r)   r*   remove_negative_resample   s    r^   c              	   C   s   t tj| |||d|tjjS )z Remove Absolute (resample)
    xlabel = "Max fraction of features removed"
    ylabel = "1 - R^2"
    transform = "one_minus"
    sort_order = 15
    r
   )r@   r   r\   rG   rH   rI   rB   r)   r)   r*   remove_absolute_resample__r2   s    r_   c              	   C   s   t tj| |||d|tjjS )z Remove Absolute (resample)
    xlabel = "Max fraction of features removed"
    ylabel = "1 - ROC AUC"
    transform = "one_minus"
    sort_order = 15
    r
   )r@   r   r\   rG   rH   rL   rB   r)   r)   r*   !remove_absolute_resample__roc_auc  s    r`   c              	   C   s   t tj| |||d|tS )z Keep Positive (impute)
    xlabel = "Max fraction of features kept"
    ylabel = "Mean model output"
    transform = "identity"
    sort_order = 16
    r   r@   r   keep_imputer=   rB   r)   r)   r*   keep_positive_impute  s    rc   c              	   C   s   t tj| |||d|tS )z Keep Negative (impute)
    xlabel = "Max fraction of features kept"
    ylabel = "Negative mean model output"
    transform = "negate"
    sort_order = 17
    rE   ra   rB   r)   r)   r*   keep_negative_impute  s    rd   c              	   C   s   t tj| |||d|tjjS )z Keep Absolute (impute)
    xlabel = "Max fraction of features kept"
    ylabel = "R^2"
    transform = "identity"
    sort_order = 18
    r
   )r@   r   rb   rG   rH   rI   rB   r)   r)   r*   keep_absolute_impute__r2   s    re   c              	   C   s   t tj| |||d|tjjS )z Keep Absolute (impute)
    xlabel = "Max fraction of features kept"
    ylabel = "ROC AUC"
    transform = "identity"
    sort_order = 19
    r
   rK   rB   r)   r)   r*   keep_absolute_impute__roc_auc)  s    rf   c              	   C   s   t tj| |||d|tS )z Remove Positive (impute)
    xlabel = "Max fraction of features removed"
    ylabel = "Negative mean model output"
    transform = "negate"
    sort_order = 7
    r   r@   r   remove_imputer=   rB   r)   r)   r*   remove_positive_impute2  s    ri   c              	   C   s   t tj| |||d|tS )z Remove Negative (impute)
    xlabel = "Max fraction of features removed"
    ylabel = "Mean model output"
    transform = "identity"
    sort_order = 8
    rE   rg   rB   r)   r)   r*   remove_negative_impute;  s    rj   c              	   C   s   t tj| |||d|tjjS )z Remove Absolute (impute)
    xlabel = "Max fraction of features removed"
    ylabel = "1 - R^2"
    transform = "one_minus"
    sort_order = 9
    r
   )r@   r   rh   rG   rH   rI   rB   r)   r)   r*   remove_absolute_impute__r2D  s    rk   c              	   C   s   t tj| |||d|tjjS )z Remove Absolute (impute)
    xlabel = "Max fraction of features removed"
    ylabel = "1 - ROC AUC"
    transform = "one_minus"
    sort_order = 9
    r
   rS   rB   r)   r)   r*   remove_absolute_impute__roc_aucM  s    rl   c              	   C   s   t tj| |||d|tS )z Keep Positive (retrain)
    xlabel = "Max fraction of features kept"
    ylabel = "Mean model output"
    transform = "identity"
    sort_order = 6
    r   r@   r   Zkeep_retrainr=   rB   r)   r)   r*   keep_positive_retrainV  s    rn   c              	   C   s   t tj| |||d|tS )z Keep Negative (retrain)
    xlabel = "Max fraction of features kept"
    ylabel = "Negative mean model output"
    transform = "negate"
    sort_order = 7
    rE   rm   rB   r)   r)   r*   keep_negative_retrain_  s    ro   c              	   C   s   t tj| |||d|tS )z Remove Positive (retrain)
    xlabel = "Max fraction of features removed"
    ylabel = "Negative mean model output"
    transform = "negate"
    sort_order = 11
    r   r@   r   Zremove_retrainr=   rB   r)   r)   r*   remove_positive_retrainh  s    rq   c              	   C   s   t tj| |||d|tS )z Remove Negative (retrain)
    xlabel = "Max fraction of features removed"
    ylabel = "Mean model output"
    transform = "identity"
    sort_order = 12
    rE   rp   rB   r)   r)   r*   remove_negative_retrainq  s    rr   c           
         s:    fdd}t d|jd |}	|	t|||	||fS )Nc           
         s|    dkrt t||}n t|| }t t||  }	t |	t |dkdt j	}	|	|||||||
S )Nr
   r   )
r   abs__strip_listoneslenminimumarraysumastypeint)
fcountr#   r$   r%   r6   r7   r8   r   AZnmaskattribution_signmeasurer   summary_functionr)   r*   r:   |  s"    $        z%__run_measure.<locals>.score_functionr
   r   __intlogspacer   r;   )
r   r   r   r   r   r   rC   r   r:   fcountsr)   r~   r*   r@   z  s    r@   c                 C   s   t tj| |||tjj|S )z Batch Remove Absolute (retrain)
    xlabel = "Fraction of features removed"
    ylabel = "1 - R^2"
    transform = "one_minus"
    sort_order = 13
    )__run_batch_abs_metricr   batch_remove_retrainrG   rH   rI   rB   r)   r)   r*   !batch_remove_absolute_retrain__r2  s    r   c                 C   s   t tj| |||tjj|S )z Batch Keep Absolute (retrain)
    xlabel = "Fraction of features kept"
    ylabel = "R^2"
    transform = "identity"
    sort_order = 13
    )r   r   batch_keep_retrainrG   rH   rI   rB   r)   r)   r*   batch_keep_absolute_retrain__r2  s    r   c                 C   s   t tj| |||tjj|S )z Batch Remove Absolute (retrain)
    xlabel = "Fraction of features removed"
    ylabel = "1 - ROC AUC"
    transform = "one_minus"
    sort_order = 13
    )r   r   r   rG   rH   rL   rB   r)   r)   r*   &batch_remove_absolute_retrain__roc_auc  s    r   c                 C   s   t tj| |||tjj|S )z Batch Keep Absolute (retrain)
    xlabel = "Fraction of features kept"
    ylabel = "ROC AUC"
    transform = "identity"
    sort_order = 13
    )r   r   r   rG   rH   rL   rB   r)   r)   r*   $batch_keep_absolute_retrain__roc_auc  s    r   c           	         s8    fdd}t d|jd |}|t|||||fS )Nc                    sr   t t||}t t||  t j}t t||}	t t||  t j}
||
||||||	 
S r4   )r   rs   rt   ru   rv   rz   r{   )r|   r#   r$   r%   r6   r7   r8   ZA_trainZnkeep_trainZA_testZ
nkeep_testlossmetricr   r)   r*   r:     s             z.__run_batch_abs_metric.<locals>.score_functionr
   r   r   )	r   r   r   r   r   r   rC   r:   r   r)   r   r*   r     s    r   
   r   /tmpc	              
      s  t j }	t jd g }
tt|   tt|  }t|D ]"t	t| ||d\ dd
t||jg d }tj
||d }tj|rt|d}t|W 5 Q R X n4|  t|d}t| W 5 Q R X d
|j|t|t|t|g f	d	d
}tkr`|
|tt| qJ|
|d qJt j|	 t |
dS )z! Test an explanation method.
    r   r   Zmodel_cache__v__z.picklerbwbr&   c                    sb    fdd}d kr*	|S g }D ] }| |	| q2t|S d S )Nc                    s   t kr | t < t  S r4   )_attribution_cache)ZX_inner)r7   attr_keyr)   r*   cached_attr_function  s    z;__score_method.<locals>.score.<locals>.cached_attr_function)r   r   rx   )r7   r   Zscoresf	r$   r#   r   r   r"   r'   r:   r6   r%   )r7   r*   score  s    z__score_method.<locals>.scoreNr
   )r   r   r   hashlibsha256r   flatten	hexdigestr   r   joinr   __name__ospathisfileopenpickleloadr   dumpstrr   r   r   r	   rx   r   )r   r   r   r   r:   r   Znrepsr   	cache_dirr    r!   Z	data_hashZmodel_idZ
cache_filer   r   r)   r   r*   r;     s,    
($
r;   c                 C   sL   |t kr| tkrtS |tkr(| tkr(tS tat ata| a|a | |atS d S r4   )
__cache_X0
__cache_f0__cache0
__cache_X1
__cache_f1__cache1)r   r   r)   r)   r*   __check_cache  s    r   c                 C   s:   t t | ||  t jdd|ddd  d  t jS )Nr
   r   T)Zendpoint	   )r   uniqueroundZlogspacerz   r{   )r(   endcountr)   r)   r*   r   	  s    r   c                 C   s   t | dr| j} | S )z* Converts DataFrames to numpy arrays.
    values)hasattrr   )r   r)   r)   r*   r     s    
r   c                 C   s   t | tr| d S | S dS )zf This assumes that if you have a list of outputs you just want the second one (the second class).
    r   N)
isinstancelist)attrsr)   r)   r*   rt     s    
rt   c           
      C   s   d}d}t ||f}|j t || }d|dddf< ||dd< tdddD ]}d||df< |||< qR||d< |  }	|	|| |	S )Ni@B    r   r
   i  )r   zerosr   ru   r   r   )
r   Zval00Zval01Zval11NMr   r   r"   r'   r)   r)   r*   
_fit_human  s    
r   c           
      C   s  t |  dkstdt d}|s\|s\t dddg}t dddgg|dd d f< nn|s|rt dddg}t dddgg|dd d f< n6|r|rt dddg}t dddgg|dd d f< t|ddd	}tt||| }||}	d
||	dd d f ffS )Nr
   JHuman agreement metrics are only for use with the human_agreement dataset!r   r   r.   r,          @      @r   r   human	r   rs   maxAssertionErrorr   rx   r   r   r	   
r   r   r   ZfeverZcoughr$   Zhuman_consensusr'   r7   Zmethods_attrsr)   r)   r*   
_human_and,  s    
  r   c                 C   s   t | ||ddS )a   AND (false/false)

    This tests how well a feature attribution method agrees with human intuition
    for an AND operation combined with linear effects. This metric deals
    specifically with the question of credit allocation for the following function
    when all three inputs are true:
    if fever: +2 points
    if cough: +2 points
    if fever and cough: +6 points

    transform = "identity"
    sort_order = 0
    Fr   r   r   r   r   r)   r)   r*   human_and_00B  s    r   c                 C   s   t | ||ddS )a   AND (false/true)

    This tests how well a feature attribution method agrees with human intuition
    for an AND operation combined with linear effects. This metric deals
    specifically with the question of credit allocation for the following function
    when all three inputs are true:
    if fever: +2 points
    if cough: +2 points
    if fever and cough: +6 points

    transform = "identity"
    sort_order = 1
    FTr   r   r)   r)   r*   human_and_01R  s    r   c                 C   s   t | ||ddS )a   AND (true/true)

    This tests how well a feature attribution method agrees with human intuition
    for an AND operation combined with linear effects. This metric deals
    specifically with the question of credit allocation for the following function
    when all three inputs are true:
    if fever: +2 points
    if cough: +2 points
    if fever and cough: +6 points

    transform = "identity"
    sort_order = 2
    Tr   r   r)   r)   r*   human_and_11b  s    r   c           
      C   s  t |  dkstdt d}|s\|s\t dddg}t dddgg|dd d f< nn|s|rt dddg}t dddgg|dd d f< n6|r|rt dddg}t dddgg|dd d f< t|ddd	}tt||| }||}	d
||	dd d f ffS )Nr
   r   r   r.   r,          @r      r   r   r   r   r)   r)   r*   	_human_ors  s    
  r   c                 C   s   t | ||ddS )a   OR (false/false)

    This tests how well a feature attribution method agrees with human intuition
    for an OR operation combined with linear effects. This metric deals
    specifically with the question of credit allocation for the following function
    when all three inputs are true:
    if fever: +2 points
    if cough: +2 points
    if fever or cough: +6 points

    transform = "identity"
    sort_order = 0
    Fr   r   r)   r)   r*   human_or_00  s    r   c                 C   s   t | ||ddS )a   OR (false/true)

    This tests how well a feature attribution method agrees with human intuition
    for an OR operation combined with linear effects. This metric deals
    specifically with the question of credit allocation for the following function
    when all three inputs are true:
    if fever: +2 points
    if cough: +2 points
    if fever or cough: +6 points

    transform = "identity"
    sort_order = 1
    FTr   r   r)   r)   r*   human_or_01  s    r   c                 C   s   t | ||ddS )a   OR (true/true)

    This tests how well a feature attribution method agrees with human intuition
    for an OR operation combined with linear effects. This metric deals
    specifically with the question of credit allocation for the following function
    when all three inputs are true:
    if fever: +2 points
    if cough: +2 points
    if fever or cough: +6 points

    transform = "identity"
    sort_order = 2
    Tr   r   r)   r)   r*   human_or_11  s    r   c           
      C   s  t |  dkstdt d}|s\|s\t dddg}t dddgg|dd d f< nn|s|rt dddg}t dddgg|dd d f< n6|r|rt dddg}t dddgg|dd d f< t|ddd	}tt||| }||}	d
||	dd d f ffS )Nr
   r   r   r.   r,   r   r   r      r   r   r   r)   r)   r*   
_human_xor  s    
  r   c                 C   s   t | ||ddS )a   XOR (false/false)

    This tests how well a feature attribution method agrees with human intuition
    for an eXclusive OR operation combined with linear effects. This metric deals
    specifically with the question of credit allocation for the following function
    when all three inputs are true:
    if fever: +2 points
    if cough: +2 points
    if fever or cough but not both: +6 points

    transform = "identity"
    sort_order = 3
    Fr   r   r)   r)   r*   human_xor_00  s    r   c                 C   s   t | ||ddS )a   XOR (false/true)

    This tests how well a feature attribution method agrees with human intuition
    for an eXclusive OR operation combined with linear effects. This metric deals
    specifically with the question of credit allocation for the following function
    when all three inputs are true:
    if fever: +2 points
    if cough: +2 points
    if fever or cough but not both: +6 points

    transform = "identity"
    sort_order = 4
    FTr   r   r)   r)   r*   human_xor_01  s    r   c                 C   s   t | ||ddS )a   XOR (true/true)

    This tests how well a feature attribution method agrees with human intuition
    for an eXclusive OR operation combined with linear effects. This metric deals
    specifically with the question of credit allocation for the following function
    when all three inputs are true:
    if fever: +2 points
    if cough: +2 points
    if fever or cough but not both: +6 points

    transform = "identity"
    sort_order = 5
    Tr   r   r)   r)   r*   human_xor_11  s    r   c           
      C   s  t |  dkstdt d}|s\|s\t dddg}t dddgg|dd d f< nn|s|rt dddg}t dddgg|dd d f< n6|r|rt dddg}t dddgg|dd d f< t|ddd}tt||| }||}	d	||	dd d f ffS )
Nr
   r   r   r.   r,   r   r   r   r   r   r   r)   r)   r*   
_human_sum  s    
  r   c                 C   s   t | ||ddS )aj   SUM (false/false)

    This tests how well a feature attribution method agrees with human intuition
    for a SUM operation. This metric deals
    specifically with the question of credit allocation for the following function
    when all three inputs are true:
    if fever: +2 points
    if cough: +2 points

    transform = "identity"
    sort_order = 0
    Fr   r   r)   r)   r*   human_sum_00  s    r   c                 C   s   t | ||ddS )ai   SUM (false/true)

    This tests how well a feature attribution method agrees with human intuition
    for a SUM operation. This metric deals
    specifically with the question of credit allocation for the following function
    when all three inputs are true:
    if fever: +2 points
    if cough: +2 points

    transform = "identity"
    sort_order = 1
    FTr   r   r)   r)   r*   human_sum_01&  s    r   c                 C   s   t | ||ddS )ah   SUM (true/true)

    This tests how well a feature attribution method agrees with human intuition
    for a SUM operation. This metric deals
    specifically with the question of credit allocation for the following function
    when all three inputs are true:
    if fever: +2 points
    if cough: +2 points

    transform = "identity"
    sort_order = 2
    Tr   r   r)   r)   r*   human_sum_115  s    r   )r>   )r>   )r>   )r>   )r>   )r>   )r>   )r>   )r>   )r>   )r>   )r>   )r>   )r>   )r>   )r>   )r>   )r>   )r>   )r>   )r>   )r>   )r>   )r>   )r>   )r>   )r>   )r>   )r>   )r>   )r>   )r>   )r   r   r   )X r   r   r   Z
explainersr   r   r   r	   rG   numpyr   copy	functoolsr   r   r   Zsklearn.model_selectionr   	ExceptionZsklearn.cross_validationr+   r5   r<   r=   rD   rF   rJ   rM   rP   rQ   rR   rT   rW   rX   rY   rZ   r]   r^   r_   r`   rc   rd   re   rf   ri   rj   rk   rl   rn   ro   rq   rr   r@   r   r   r   r   r   r   r;   r   r   r   r   r   r   r   r   r   rt   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r)   r)   r)   r*   <module>   s   "$
	
	
	
	
	
	
	
	
	
	
	
	
	
	
	
	
	
	
	
	
	
	
	
	
	
	
	
	
	
	
	
	
5