
\c           @   sd
  d  Z  d d l m Z d d l Z d d l Z d d l Z d d l Z d d l m Z d d l	 Z	 d d l
 Z d d l m Z m Z d d l m Z d d l m Z d d l m Z d d	 l m Z d d
 l m Z d d l m Z d d l m Z d d l m Z d d l m Z d d l m Z d d l m Z d d l m Z d d l m Z d d l m Z d d l m  Z  d d l m! Z! d d l" m# Z# m$ Z$ d d l% m& Z& d d l% m' Z' d d l% m( Z( d d l% m) Z) d d l% m* Z* d d l% m+ Z+ d d l% m, Z, d d l% m- Z- d d l% m. Z. d d  l% m/ Z/ d d! l% m0 Z0 d d" l% m1 Z1 d d# l% m2 Z2 d d$ l3 m4 Z4 d d% l3 m5 Z5 d d& l6 m7 Z7 d d' l6 m8 Z8 d d( l6 m9 Z9 d d) l6 m: Z: d d* l; m< Z< d d+ l; m= Z= d d, l; m> Z> d d- l; m? Z? d d. l; m@ Z@ d d/ l; mA ZA d d0 l; mB ZB d d1 lC mD ZD d d2 lE mF ZF mG ZG mH ZH d d3 lE mI ZI mJ ZJ d d4 lK mL ZL d d5 lM mN ZN d d6 lO mP ZP d d7 lQ mR ZR d d8 lS mT ZT d d9 lU mV ZV d d: lW mX ZY d d; lZ m[ Z[ d d< lZ m\ Z\ d d= l] m^ Z^ d d> l_ m` Z` d d? l6 ma Za d d@ l6 mb Zb d dA lc md Zd d dB l% me Ze y ef Wn eg k
 rd Zf n XdC e[ f dD     YZi dE ei f dF     YZj dG e[ f dH     YZk dI ek f dJ     YZl dK em f dL     YZn e jo d  Zp e ep  Zq e jr dO dO dP dP dN dN dQ dQ dR dR g
  Zs e jr dP dP dP dN dN dN dQ dQ dQ dQ g
  Zt e e ju dS   Zv e	 jw jx dT  dU    Zy e	 jw jx dV  dW    Zz e	 jw jx dT  dX    Z{ dY   Z| dZ   Z} d[   Z~ d\   Z e	 jw jx dT  d]    Z e	 jw jx d^  e	 jw jx dT  d_     Z d`   Z e	 jw jx dT  da    Z e	 jw jx dT  db    Z dc   Z dd   Z de   Z df   Z dg   Z dh   Z di   Z dj   Z e	 jw jx dk  e	 jw jx dl  e	 jw jx dT  dm      Z e	 jw jx dk  e	 jw jx dl  e	 jw jx dT  dn      Z e	 jw jx dk  e	 jw jx dl  e	 jw jx dT  do      Z e	 jw jx dk  e	 jw jx dl  e	 jw jx dT  dp      Z e	 jw jx dk  e	 jw jx dl  e	 jw jx dT  dq      Z e	 jw jx d^  e	 jw jx dT  dr     Z e	 jw jx dT  ds    Z dt   Z du   Z dv   Z dw   Z dx   Z dy   Z e	 jw jx dz  d{    Z d|   Z d}   Z d~   Z e	 jw jx dz  d    Z d   Z d   Z d   Z d   Z d   Z d   Z e	 jw jx dk  e	 jw jx dl  e	 jw jx dT  d      Z e	 jw jx d  e	 jw jx dT  d     Z e	 jw jx dk  e	 jw jx dl  e	 jw jx d  e	 jw jx dT  d       Z d   Z e	 jw jx dk  e	 jw jx dl  e	 jw jx dT  d      Z e	 jw jx dT  d    Z e	 jw jx d^  e	 jw jx dT  d     Z d   Z d S(   s   Test the validation modulei(   t   divisionN(   t   sleep(   t
   coo_matrixt
   csr_matrix(   t   FitFailedWarning(   t   FailingClassifier(   t   assert_true(   t   assert_false(   t   assert_equal(   t   assert_almost_equal(   t   assert_raises(   t   assert_raise_message(   t   assert_warns(   t   assert_warns_message(   t   assert_no_warnings(   t   assert_raises_regex(   t   assert_greater(   t   assert_less(   t   assert_array_almost_equal(   t   assert_array_equal(   t   CheckingClassifiert   MockDataFrame(   t   cross_val_score(   t   cross_val_predict(   t   cross_validate(   t   permutation_test_score(   t   KFold(   t   StratifiedKFold(   t   LeaveOneOut(   t   LeaveOneGroupOut(   t   LeavePGroupsOut(   t
   GroupKFold(   t   GroupShuffleSplit(   t   learning_curve(   t   validation_curve(   t   _check_is_permutation(   t   _fit_and_score(   t   make_regression(   t   load_boston(   t	   load_iris(   t   load_digits(   t   explained_variance_score(   t   make_scorer(   t   accuracy_score(   t   confusion_matrix(   t   precision_recall_fscore_support(   t   precision_score(   t   r2_score(   t   check_scoring(   t   Ridget   LogisticRegressiont   SGDClassifier(   t   PassiveAggressiveClassifiert   RidgeClassifier(   t   KNeighborsClassifier(   t   SVC(   t   KMeans(   t   SimpleImputer(   t   LabelEncoder(   t   Pipeline(   t	   cStringIO(   t   BaseEstimator(   t   clone(   t   OneVsRestClassifier(   t   shuffle(   t   make_classification(   t   make_multilabel_classification(   t   OneTimeSplitter(   t   GridSearchCVt   MockImprovingEstimatorc           B   sD   e  Z d  Z d   Z d d  Z d   Z d d d  Z d   Z RS(   s+   Dummy classifier to test the learning curvec         C   s   | |  _  d |  _ d  |  _ d  S(   Ni    (   t   n_max_train_sizest   train_sizest   Nonet   X_subset(   t   selfRF   (    (    sL   lib/python2.7/site-packages/sklearn/model_selection/tests/test_validation.pyt   __init__]   s    		c         C   s   | |  _  | j d |  _ |  S(   Ni    (   RI   t   shapeRG   (   RJ   RI   t   y_subset(    (    sL   lib/python2.7/site-packages/sklearn/model_selection/tests/test_validation.pyt   fitb   s    	c         C   s
   t   d  S(   N(   t   NotImplementedError(   RJ   t   X(    (    sL   lib/python2.7/site-packages/sklearn/model_selection/tests/test_validation.pyt   predictg   s    c         C   s?   |  j  |  r' d t |  j  |  j St |  j  |  j Sd  S(   Ng       @(   t   _is_training_datat   floatRG   RF   (   RJ   RP   t   Y(    (    sL   lib/python2.7/site-packages/sklearn/model_selection/tests/test_validation.pyt   scorej   s    c         C   s   | |  j  k S(   N(   RI   (   RJ   RP   (    (    sL   lib/python2.7/site-packages/sklearn/model_selection/tests/test_validation.pyRR   q   s    N(	   t   __name__t
   __module__t   __doc__RK   RH   RN   RQ   RU   RR   (    (    (    sL   lib/python2.7/site-packages/sklearn/model_selection/tests/test_validation.pyRE   [   s   		t!   MockIncrementalImprovingEstimatorc           B   s,   e  Z d  Z d   Z d   Z d d  Z RS(   s*   Dummy classifier that provides partial_fitc         C   s#   t  t |   j |  d  |  _ d  S(   N(   t   superRY   RK   RH   t   x(   RJ   RF   (    (    sL   lib/python2.7/site-packages/sklearn/model_selection/tests/test_validation.pyRK   w   s    c         C   s   |  j  | k S(   N(   R[   (   RJ   RP   (    (    sL   lib/python2.7/site-packages/sklearn/model_selection/tests/test_validation.pyRR   |   s    c         K   s'   |  j  | j d 7_  | d |  _ d  S(   Ni    (   RG   RL   R[   (   RJ   RP   t   yt   params(    (    sL   lib/python2.7/site-packages/sklearn/model_selection/tests/test_validation.pyt   partial_fit   s    N(   RV   RW   RX   RK   RR   RH   R^   (    (    (    sL   lib/python2.7/site-packages/sklearn/model_selection/tests/test_validation.pyRY   u   s   		t   MockEstimatorWithParameterc           B   sD   e  Z d  Z d d  Z d   Z d   Z d d d  Z d   Z RS(   s-   Dummy classifier to test the validation curveg      ?c         C   s   d  |  _ | |  _ d  S(   N(   RH   RI   t   param(   RJ   R`   (    (    sL   lib/python2.7/site-packages/sklearn/model_selection/tests/test_validation.pyRK      s    	c         C   s   | |  _  | j d |  _ |  S(   Ni    (   RI   RL   RG   (   RJ   RI   RM   (    (    sL   lib/python2.7/site-packages/sklearn/model_selection/tests/test_validation.pyRN      s    	c         C   s
   t   d  S(   N(   RO   (   RJ   RP   (    (    sL   lib/python2.7/site-packages/sklearn/model_selection/tests/test_validation.pyRQ      s    c         C   s!   |  j  |  r |  j Sd |  j S(   Ni   (   RR   R`   (   RJ   RP   R\   (    (    sL   lib/python2.7/site-packages/sklearn/model_selection/tests/test_validation.pyRU      s    c         C   s   | |  j  k S(   N(   RI   (   RJ   RP   (    (    sL   lib/python2.7/site-packages/sklearn/model_selection/tests/test_validation.pyRR      s    N(	   RV   RW   RX   RK   RN   RQ   RH   RU   RR   (    (    (    sL   lib/python2.7/site-packages/sklearn/model_selection/tests/test_validation.pyR_      s   		t%   MockEstimatorWithSingleFitCallAllowedc           B   s    e  Z d  Z d   Z d   Z RS(   s<   Dummy classifier that disallows repeated calls of fit methodc         C   s>   t  t |  d  d  t |  _ t t |   |   j | |  S(   Nt   fit_called_s   fit is called the second time(   R   t   hasattrt   TrueRb   RZ   t   typeRN   (   RJ   RI   RM   (    (    sL   lib/python2.7/site-packages/sklearn/model_selection/tests/test_validation.pyRN      s
    	c         C   s
   t   d  S(   N(   RO   (   RJ   RP   (    (    sL   lib/python2.7/site-packages/sklearn/model_selection/tests/test_validation.pyRQ      s    (   RV   RW   RX   RN   RQ   (    (    (    sL   lib/python2.7/site-packages/sklearn/model_selection/tests/test_validation.pyRa      s   	t   MockClassifierc        
   B   se   e  Z d  Z d e d  Z d d d d d d d d d d 	 Z d   Z d d d  Z e d  Z	 RS(   s-   Dummy classifier to test the cross-validationi    c         C   s   | |  _  | |  _ d  S(   N(   t   at   allow_nd(   RJ   Rg   Rh   (    (    sL   lib/python2.7/site-packages/sklearn/model_selection/tests/test_validation.pyRK      s    	c         C   s  | |  _  | |  _ |	 |  _ |
 d
 k	 r4 |
 |   n  |  j rX | j t |  d  } n  | j d k r |  j r t d   n  | d
 k	 r t	 | j
 d | j
 d k d j | j
 d | j
 d   n  | d
 k	 r$t	 | j
 d t t j t   k d j | j
 d t t j t     n  | d
 k	 rtd } t	 | j
 d | j
 d k | j | j
 d | j
 d   n  | d
 k	 rd } t	 | j
 t j
 k | j | j
 d | j
 d	 t j
 d t j
 d	   n  |  S(   s   The dummy arguments are to test that this fit function can
        accept non-array arguments through cross-validation, such as:
            - int
            - str (this is actually array-like)
            - object
            - function
        ii   s   X cannot be di    sK   MockClassifier extra fit_param sample_weight.shape[0] is {0}, should be {1}sI   MockClassifier extra fit_param class_prior.shape[0] is {0}, should be {1}sR   MockClassifier extra fit_param sparse_sample_weight.shape[0] is {0}, should be {1}sU   MockClassifier extra fit_param sparse_param.shape is ({0}, {1}), should be ({2}, {3})i   N(   t	   dummy_intt	   dummy_strt	   dummy_objRH   Rh   t   reshapet   lent   ndimt
   ValueErrorR   RL   t   formatt   npt   uniqueR\   t   P_sparse(   RJ   RP   RT   t   sample_weightt   class_priort   sparse_sample_weightt   sparse_paramRi   Rj   Rk   t   callbackt   fmt(    (    sL   lib/python2.7/site-packages/sklearn/model_selection/tests/test_validation.pyRN      s<    
				
"
$
c         C   s8   |  j  r$ | j t |  d  } n  | d  d   d f S(   Nii    (   Rh   Rl   Rm   (   RJ   t   T(    (    sL   lib/python2.7/site-packages/sklearn/model_selection/tests/test_validation.pyRQ      s    	c         C   s   d d t  j |  j  S(   Ng      ?i   (   Rq   t   absRg   (   RJ   RP   RT   (    (    sL   lib/python2.7/site-packages/sklearn/model_selection/tests/test_validation.pyRU      s    c         C   s   i |  j  d 6|  j d 6S(   NRg   Rh   (   Rg   Rh   (   RJ   t   deep(    (    sL   lib/python2.7/site-packages/sklearn/model_selection/tests/test_validation.pyt
   get_params   s    N(
   RV   RW   RX   t   FalseRK   RH   RN   RQ   RU   R}   (    (    (    sL   lib/python2.7/site-packages/sklearn/model_selection/tests/test_validation.pyRf      s   		*	i
   i   i    i   i   i   i   s"   ignore: You should specify a valuec          C   s  t    }  x t d d  D] } | |  _ t |  t t  } t | |  j t t   t j	 t t d  d  d  g  } t |  t
 |  } t | |  j t
 |   t |  t
 t  } t | |  j t
 t   t |  t
 |  } t | |  j t
 |   q Wd   } t d |  }  t |  t j   t j    } t d |  }  t |  t t j    } t t t |  t t d d t d  d   d  d   t j f } t  d	 t  }  t |  | t  } t  d	 t  }  t t t |  | t d
 d d  S(   Nii
   ic         S   s   t  |  t  S(   N(   t
   isinstancet   list(   R[   (    (    sL   lib/python2.7/site-packages/sklearn/model_selection/tests/test_validation.pyt   <lambda>	  s    t   check_Xt   check_yt   scoringt   sklearnRh   t   error_scoret   raise(   Rf   t   rangeRg   R   RP   t   y2R   RU   Rq   t   column_stackt   X_sparseR   t   tolistR
   Ro   t   newaxisRd   R~   (   t   clfRg   t   scorest   multioutput_yt
   list_checkt   X_3d(    (    sL   lib/python2.7/site-packages/sklearn/model_selection/tests/test_validation.pyt   test_cross_val_score   s0    		"	"s%   ignore:You should specify a value forc          C   s]   t  d t  \ }  } t d d  } t | d i d d g d 6} t | |  | d d	 d  S(
   Nt
   return_X_yt   gammat   autot
   param_gridi   i
   t   Ct   n_jobsi   (   R'   Rd   R7   RD   R   (   RP   R\   R   t   grid(    (    sL   lib/python2.7/site-packages/sklearn/model_selection/tests/test_validation.pyt   test_cross_validate_many_jobs  s    c          C   s  t  d d  \ }  } t   } d } t t | t | |  | d t t  t t  f t t | t | |  | d t t  f t t | d t | |  | d d t t | d t | |  | d d t t | t | |  | d t t  g g d } t t d	 t | |  | d t   t t | t | |  | d d
 t t	  } t t d t | |  | d | t t d t | |  | d i | d 6t t
  } t t d t t d d  |  | d | t t d t t d d  |  | d i | d 6t t d t t   |  | d d d  S(   Nt   random_statei    s   .*must be unique strings.*R   s   Empty list.*s   Duplicate.*t   f1_microsG   .*should either be.*string or callable.*for single.*.*dict.*for multi.*s   An empty dicti   sJ   Classification metrics can't handle a mix of binary and continuous targetst   foos!   scoring must return a number, gotR   t   scales#   'mse' is not a valid scoring value.t   mse(    (   R   R   (   RA   Rf   R   Ro   R   R*   R.   R+   t   dictR-   R,   R7   (   RP   R\   t	   estimatort   error_message_regexpt   multiclass_scorert   multivalued_scorer(    (    sL   lib/python2.7/site-packages/sklearn/model_selection/tests/test_validation.pyt)   test_cross_validate_invalid_scoring_param'  sR    										c          C   s5  t  d d  }  t d d d d  \ } } t d d  } t d d d d  \ } } t d d d d  } x| | | f | | | f f D]\ } } }	 t |	 d	  }
 t |	 d
  } g  } g  } g  } g  } g  } x |  j | |  D] \ } } t |  j | | | |  }	 | j	 |
 |	 | | | |   | j	 | |	 | | | |   | j	 |
 |	 | | | |   | j	 | |	 | | | |   | j	 |	  q Wt
 j |  } t
 j |  } t
 j |  } t
 j |  } t
 j |  } | | | | | f } t |	 | | |  t |	 | | |  q Wd  S(   Nt   n_splitsi   t	   n_samplesi   R   i    t   kernelt   lineart   neg_mean_squared_errort   r2(   R   R%   R1   RA   R7   R0   t   splitR>   RN   t   appendRq   t   arrayt"   check_cross_validate_single_metrict!   check_cross_validate_multi_metric(   t   cvt   X_regt   y_regt   regt   X_clft   y_clfR   RP   R\   t   estt
   mse_scorert	   r2_scorert   train_mse_scorest   test_mse_scorest   train_r2_scorest   test_r2_scorest   fitted_estimatorst   traint   testR   (    (    sL   lib/python2.7/site-packages/sklearn/model_selection/tests/test_validation.pyt   test_cross_validaten  s:    . !!!!	c       
   C   s   t  d d  \ }  } t   } i  } x< t t d g D]+ } t t | |  | d | d d | | <q4 Wd j d  } t t | | d j	 d  } t
 j | | t d  s t  d | t k s t  d  S(	   NR   i    t   warnt   return_train_scoreR   i   s   You are accessing a training score ({!r}), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=Truet   train_score(   RA   Rf   R~   Rd   R   R   Rp   R   t   FutureWarningt   getRq   t   allcloset   AssertionError(   RP   R\   R   t   resultt   valt   msgR   (    (    sL   lib/python2.7/site-packages/sklearn/model_selection/tests/test_validation.pyt+   test_cross_validate_return_train_score_warn  s    			 c         C   s  | \ } } } } } xet  d f t d f f D]K\ }	 }
 |	 rx t |  | | d d d d d t  } t | d |  n$ t |  | | d d d d d t } t | t  s t  t t |  |
  t | d	 |  |	 rt |  | | d d d d
 g d t  } t | d | t   n' t |  | | d d d d
 g d t } t | t  sUt  t t |  |
  t | d |  q. Wt |  | | d d d d d t  } xL t	 | d  D]: \ } } t
 | j | | j  t
 | j | | j  qWd  S(   Ni   i   R   i   R   R   R   R   t
   test_scoreR   t   train_r2t   test_r2t   return_estimatorR   (   Rd   R~   R   R   R   R   R   R   Rm   t	   enumerateR	   t   coef_t
   intercept_(   R   RP   R\   R   R   R   R   R   R   R   t   dict_lent   mse_scores_dictt   r2_scores_dictt   kR   (    (    sL   lib/python2.7/site-packages/sklearn/model_selection/tests/test_validation.pyR     s>    %	

	
	
		c         C   sI  | \ } } } } } d i t  t  d 6d d 6f }	 t d  }
 |
 j t d   } xt t f D]} x|	 D]} | r t |  | | d	 d
 d | d t } t | d |  t | d |  n$ t |  | | d	 d
 d | d t } t | t	  s t
  t t | j    | r| n |
  t | d |  t | d |  t | d  t j k sdt
  t | d  t j k st
  t | d  t j k st
  t | d  t j k st
  t j | d d k  st
  t j | d d k   st
  t j | d d k  st
  t j | d d k   sp t
  qp Wqc Wd  S(   NR   R   R   t   test_neg_mean_squared_errort   fit_timet
   score_timeR   t   train_neg_mean_squared_errorR   i   R   R   i    i
   (   R   R   (   R   R   R   R   (   R   R   (   R*   R/   t   sett   unionRd   R~   R   R   R   R   R   R   t   keysRe   Rq   t   ndarrayt   all(   R   RP   R\   R   R   R   R   R   R   t   all_scoringt   keys_sans_traint   keys_with_trainR   R   t
   cv_results(    (    sL   lib/python2.7/site-packages/sklearn/model_selection/tests/test_validation.pyR     sH    		

	c          C   s   t  d d d d d d  \ }  } t d d  } t   t d  t   t   g } x^ | D]V } t t d	 t d
 | d |  d | d | t t d	 t	 d
 | d |  d | d | qX Wd  S(   NR   i   t	   n_classesi   R   i    R   R   s*   The 'groups' parameter should not be None.R   RP   R\   R   (
   RA   R7   R   R   R   R    R   Ro   R   R   (   RP   R\   R   t	   group_cvsR   (    (    sL   lib/python2.7/site-packages/sklearn/model_selection/tests/test_validation.pyt#   test_cross_val_score_predict_groups  s    !s(   ignore: Using or importing the ABCs fromc             s   t  t  f g }  y- d d l m } m } |  j | | f  Wn t k
 rO n Xxp |  D]h \      t   t  } }   f d   }  f d   } t d | d |  } t	 | | |  qW Wd  S(   Ni(   t   Seriest	   DataFramec            s   t  |     S(   N(   R   (   R[   (   t   InputFeatureType(    sL   lib/python2.7/site-packages/sklearn/model_selection/tests/test_validation.pyR   .  s    c            s   t  |     S(   N(   R   (   R[   (   t
   TargetType(    sL   lib/python2.7/site-packages/sklearn/model_selection/tests/test_validation.pyR   /  s    R   R   (
   R   t   pandasR   R   R   t   ImportErrorRP   R   R   R   (   t   typesR   R   t   X_dft   y_sert   check_dft   check_seriesR   (    (   R   R   sL   lib/python2.7/site-packages/sklearn/model_selection/tests/test_validation.pyt   test_cross_val_score_pandas   s    c          C   s  t  d d  }  t   } | j | j } } t d  } t |  | | d | } t d  } g  } x | j | |  D]o \ } } t j t	 |  d t j
 }	 t j t	 |  d t j
 }
 d |	 | <d |
 | <| j | | f  qt Wt |  | | d | } t | |  d  S(   NR   R   i   R   t   dtypei   (   R7   R'   t   datat   targetR   R   R   Rq   t   zerosRm   t   boolR   R   (   t   svmt   irisRP   R\   t   kfoldt   scores_indicest   cv_masksR   R   t
   mask_traint	   mask_testt   scores_masks(    (    sL   lib/python2.7/site-packages/sklearn/model_selection/tests/test_validation.pyt   test_cross_val_score_mask4  s    	

c          C   s   t  d d  }  t   } | j | j } } t j | | j  } t |  | |  } t  d d  }  t |  | |  } t | |  t  d d d d    }  t |  | |  } t | |  t  d d  }  t	 t
 t |  | |  t	 t
 t |  | j   |  d  S(   NR   t   precomputedR   R   R   c         S   s   t  j |  | j  S(   N(   Rq   t   dotRz   (   R[   R\   (    (    sL   lib/python2.7/site-packages/sklearn/model_selection/tests/test_validation.pyR   T  s    (   R7   R'   R   R   Rq   R  Rz   R   R   R
   Ro   R   (   R   R   RP   R\   t   linear_kernelt   score_precomputedt   score_lineart   score_callable(    (    sL   lib/python2.7/site-packages/sklearn/model_selection/tests/test_validation.pyt    test_cross_val_score_precomputedG  s    	c             s  t    }  t j d } t t j t   } t t j d g  t j d g  t j d g  f f d d } t t j	 d   } d   d  t
        f d   } i t j |  d	 6t j | d
 |  d 6| d 6| d 6  d 6 d 6 d 6| d 6} t |  t t d | d  S(   Ni    i   RL   i
   i   i*   t   42c            s4   t  |  j    t  |  j   t  |  j   d  S(   N(   R   Ri   Rj   Rk   (   R   (   t	   DUMMY_INTt	   DUMMY_OBJt	   DUMMY_STR(    sL   lib/python2.7/site-packages/sklearn/model_selection/tests/test_validation.pyt   assert_fit_paramsp  s    Rt   g      ?Ru   Rv   Rw   Ri   Rj   Rk   Rx   t
   fit_params(   i
   i   (   Rf   RP   RL   Rm   Rq   Rr   R\   R   R   t   eyet   objectt   onest   fullR   (   R   R   R   t   W_sparseRs   R  R  (    (   R  R  R  sL   lib/python2.7/site-packages/sklearn/model_selection/tests/test_validation.pyt   test_cross_val_score_fit_paramsb  s&    	9		
c             s   t    }  g      f d   } t j d t  / t |  } t |  t t d | d d } Wd  QXt | d d d g  t	    d k s t
  d  S(   Nc            s     j  |  | f  d S(   Ng      ?(   R   (   t   y_testt	   y_predict(   t   _score_func_args(    sL   lib/python2.7/site-packages/sklearn/model_selection/tests/test_validation.pyt
   score_func  s    t   recordR   R   i   g      ?(   Rf   t   warningst   catch_warningsRd   R*   R   RP   R\   R   Rm   R   (   R   R  R   RU   (    (   R  sL   lib/python2.7/site-packages/sklearn/model_selection/tests/test_validation.pyt   test_cross_val_score_score_func  s    	$c          C   s-   d d d     Y}  t  t t |    t  d  S(   Nt   BrokenEstimatorc           B   s   e  Z RS(    (   RV   RW   (    (    (    sL   lib/python2.7/site-packages/sklearn/model_selection/tests/test_validation.pyR    s   (    (   R
   t	   TypeErrorR   RP   (   R  (    (    sL   lib/python2.7/site-packages/sklearn/model_selection/tests/test_validation.pyt   test_cross_val_score_errors  s    c          C   s   t    }  t d d  } t | |  j |  j d d } t | d d d d d g d  t | |  j |  j d d	 d d } t | d d d d d g d  t | |  j |  j d d
 d d } t | d d d d d g d  d  S(   NR   R   R   i   g
ףp=
?g      ?i   R   t   accuracyt   f1_weighted(   R'   R7   R   R   R   R   (   R   R   R   t	   zo_scorest	   f1_scores(    (    sL   lib/python2.7/site-packages/sklearn/model_selection/tests/test_validation.pyt3   test_cross_val_score_with_score_func_classification  s    	c    	   	   C   s=  t  d d d d d d d d  \ }  } t   } t | |  | d	 d } t | d
 d d d d g d  t | |  | d d d	 d } t | d
 d d d d g d  t | |  | d	 d d d } t j d d d d d g  } t | | d  t t  } t | |  | d	 d d | } t | d
 d d d d g d  d  S(   NR   i   t
   n_featuresi   t   n_informativei   R   i    R   gGz?g
ףp=
?gGz?gq=
ףp?i   R   R   R   g(\؇gzGIgGz&qg\(qg)\G(   R%   R1   R   R   Rq   R   R*   R)   (	   RP   R\   R   R   t	   r2_scorest   neg_mse_scorest   expected_neg_mseR   t	   ev_scores(    (    sL   lib/python2.7/site-packages/sklearn/model_selection/tests/test_validation.pyt/   test_cross_val_score_with_score_func_regression  s    		c          C   sM  t    }  |  j } t |  } |  j } t d d  } t d  } t | | | d d d | d d \ } } } t | d	  t | d
 d  t | | | d d d | d d d t	 j
 | j  d d \ }	 }
 } |	 | k s t  | | k s t  t d d  } t d  } t | | | d d d | d d d t	 j
 | j  d d \ }	 }
 } |	 | k sgt  | | k syt  d   } t |  } t | | | d d d | d | d d \ } }
 } t | d d  t | d d  t	 j t	 j t |   d  } t | | | d d d | d d \ } } } t | d  t | d  d  S(   NR   R   i   t   n_permutationsi   R   R   R!  g?g        i   t   groupsR   i    c         S   s+   |  | k j    |  | k j    |  j d S(   Ni    (   t   sumRL   (   t   y_truet   y_pred(    (    sL   lib/python2.7/site-packages/sklearn/model_selection/tests/test_validation.pyt   custom_score  s    id   g(\?g{Gz?i   g      ?g?(   R'   R   R   R   R7   R   R   R   R	   Rq   R  t   sizeR   R*   t   modt   arangeRm   R   (   R   RP   R   R\   R   R   RU   R   t   pvaluet   score_groupt   _t   pvalue_groupt
   svm_sparset	   cv_sparseR2  t   scorer(    (    sL   lib/python2.7/site-packages/sklearn/model_selection/tests/test_validation.pyt   test_permutation_score  sB    			*$*	0!*c          C   s   t  j d d t  j j d d  }  t  j |  d d  d   f <t  j d d g |  j d d  } t d t d	 d
 d t  j  f d t	   f g  } t
 | |  | d d d  S(   Ni   R   i
   ii   i    i   t   imputert   strategyt   meant   missing_valuest
   classifierR   i   (   Rq   R5  t   float64Rl   t   nant   repeatRL   R;   R9   Rf   R   (   RP   R\   t   p(    (    sL   lib/python2.7/site-packages/sklearn/model_selection/tests/test_validation.pyt&   test_permutation_test_score_allow_nans  s    $#c          C   s   t  j d d t  j j d d  }  t  j |  d d  d   f <t  j d d g |  j d d  } t d t d	 d
 d t  j  f d t	   f g  } t
 | |  | d d d  S(   Ni   R   i
   ii   i    i   R>  R?  R@  RA  RB  R   i   (   Rq   R5  RC  Rl   RD  RE  RL   R;   R9   Rf   R   (   RP   R\   RF  (    (    sL   lib/python2.7/site-packages/sklearn/model_selection/tests/test_validation.pyt   test_cross_val_score_allow_nans  s    $#c    	      C   s  t  j d d g d d g d d g d d g d d g d d g d d g d d g d d g d d g g
  }  t  j d d g d d g d d g d d g d d g d d g d d g d d g d d g d d g g
  } t d	 d  } t t d
 d } t t d
 d } t t d
 d } t | |  | d | d d } t | |  | d | d d } t | |  | d | d d } t | d d d d d g  t | d d d d d g  t | d d d d d g  d  S(   Nii   i   i   i    i   iit   n_neighborst   averaget   microt   macrot   samplesR   R   i   g      ?g      ?g      ?gUUUUUU?g      ?g      ?g      ?g      ?g      ?g      ?g      ?g      ?(   Rq   R   R6   R*   R.   R   R	   (	   RP   R\   R   t   scoring_microt   scoring_macrot   scoring_samplest   score_microt   score_macrot   score_samples(    (    sL   lib/python2.7/site-packages/sklearn/model_selection/tests/test_validation.pyt   test_cross_val_score_multilabel  s    3636s&   ignore: Default solver will be changeds    ignore: Default multi_class willc          C   s  t    }  |  j |  j } } t   } t   } t j |  } xO | j | |  D]; \ } } | j | | | |  | j	 | |  | | <qP Wt
 | | | d | } t | |  t
 | | |  } t t |  t |   t   } t
 | | | d | } t t |  t |   | j   }	 |	 |	 t j |	  k 9}	 t |	  }	 t
 | |	 |  } t t |  t |   t
 t   |  } t t |  t |   d d	 d     Y}
 t t t
 | | | d |
   t d t  \ } } d } t t | t
 t   | | d d d t d  d  S(
   NR   t   BadCVc           B   s   e  Z d d d   Z RS(   c         s   sS   xL t  d  D]> } t j d d d d g  t j d d d d d	 g  f Vq Wd  S(
   Ni   i    i   i   i   i   i   i   i   (   R   Rq   R   (   RJ   RP   R\   R.  t   i(    (    sL   lib/python2.7/site-packages/sklearn/model_selection/tests/test_validation.pyR   H  s    N(   RV   RW   RH   R   (    (    (    sL   lib/python2.7/site-packages/sklearn/model_selection/tests/test_validation.pyRU  G  s   R   s   Number of classes in training fold (2) does not match total number of classes (3). Results may not be appropriate for your use case.t   methodt   predict_probai   (    (   R&   R   R   R   R1   Rq   t
   zeros_likeR   RN   RQ   R   R   R   Rm   R   t   copyt   medianR   R8   R
   Ro   R'   Rd   R   t   RuntimeWarningR2   (   t   bostonRP   R\   R   R   t   preds2R   R   t   predst   XspRU  t   warning_message(    (    sL   lib/python2.7/site-packages/sklearn/model_selection/tests/test_validation.pyt   test_cross_val_predict$  s:    						c          C   sv  t  d d d d d d  \ }  } t t   |  | d d } t | j d  t d	 t  \ }  } t t   |  | d d } t | j d  |  d  }  | d  } t t d t t	   |  | d d d t
 d  t d	 t  \ }  } t d d d d  } t | |  | d d } t | j d  t j |  } |  | | | }  } t t d t | |  | d t
 d d  d d d  S(   NR   i   R   i2   R   i    RW  t   decision_functionR   i   i   id   s   Only 1 class/es in training fold, this is not supported for decision_function with imbalanced folds. To fix this, use a cross-validation technique resulting in properly stratified foldsR   R   R   t   decision_function_shapet   ovoi  i-   s   Output shape \(599L?, 21L?\) of decision_function does not match number of classes \(7\) in fold. Irregular decision_function .*R   (   i2   (   i   i   (   i  i-   (   RA   R   R2   R   RL   R'   Rd   R   Ro   R5   R   R(   R7   Rq   t   argsortR   (   RP   R\   R_  R   t   ind(    (    sL   lib/python2.7/site-packages/sklearn/model_selection/tests/test_validation.pyt.   test_cross_val_predict_decision_function_shapeX  s4    !		

		c          C   s   t  d d d d d d  \ }  } t t   |  | d d } t | j d  t d	 t  \ }  } t t   |  | d d } t | j d  d  S(   NR   i   R   i2   R   i    RW  RX  R   i   i   (   i2   i   (   i   i   (   RA   R   R2   R   RL   R'   Rd   (   RP   R\   R_  (    (    sL   lib/python2.7/site-packages/sklearn/model_selection/tests/test_validation.pyt*   test_cross_val_predict_predict_proba_shape  s    !		c          C   s   t  d d d d d d  \ }  } t t   |  | d d } t | j d  t d	 t  \ }  } t t   |  | d d } t | j d  d  S(   NR   i   R   i2   R   i    RW  t   predict_log_probaR   i   i   (   i2   i   (   i   i   (   RA   R   R2   R   RL   R'   Rd   (   RP   R\   R_  (    (    sL   lib/python2.7/site-packages/sklearn/model_selection/tests/test_validation.pyt.   test_cross_val_predict_predict_log_proba_shape  s    !		c    
      C   s  t    }  |  j |  j } } t |  } t j | | d  d  d  g  } t d t d d  } t | | |  } t	 | j
 d  t | | |  } t	 | j
 d  t | | |  } t | j
 d  t | | |  } t | j
 d  d   } t d |  } t | | j   | j    } t d	 |  } t | | | j    } t t   | j   | j   d
 d } t t   | | j   d
 d } | d  d   d  d   t j f } d   }	 t d |	  } t | | |  } t | j
 d  d  S(   Nit   fit_interceptR   i    i   i   c         S   s   t  |  t  S(   N(   R   R   (   R[   (    (    sL   lib/python2.7/site-packages/sklearn/model_selection/tests/test_validation.pyR     s    R   R   RW  Rc  c         S   s   |  j  d k S(   Ni   (   Rn   (   R[   (    (    sL   lib/python2.7/site-packages/sklearn/model_selection/tests/test_validation.pyR     s    (   i   (   i   i   (   i   (   i   i   (   i   (   R'   R   R   R   Rq   R   R1   R~   R   R   RL   R   R   R   R2   R   (
   R   RP   R\   R   R   R   t   predictionsR   R   t   check_3d(    (    sL   lib/python2.7/site-packages/sklearn/model_selection/tests/test_validation.pyt"   test_cross_val_predict_input_types  s6    	"	"	c             s   t  t  f g }  y- d d l m } m } |  j | | f  Wn t k
 rO n Xxp |  D]h \      t   t  } }   f d   }  f d   } t d | d |  } t	 | | |  qW Wd  S(   Ni(   R   R   c            s   t  |     S(   N(   R   (   R[   (   R   (    sL   lib/python2.7/site-packages/sklearn/model_selection/tests/test_validation.pyR     s    c            s   t  |     S(   N(   R   (   R[   (   R   (    sL   lib/python2.7/site-packages/sklearn/model_selection/tests/test_validation.pyR     s    R   R   (
   R   R   R   R   R   R   RP   R   R   R   (   R   R   R   R   R   R   R   R   (    (   R   R   sL   lib/python2.7/site-packages/sklearn/model_selection/tests/test_validation.pyt   test_cross_val_predict_pandas  s    c          C   sz   t    }  |  j |  j } } t   } i t t j | j d   d 6} t | | | d | } t	 | t j
 d   d  S(   Ni    Rv   R  i   (   R'   R   R   Rf   R   Rq   R  RL   R   R   R  (   R   RP   R\   R   R  Rg   (    (    sL   lib/python2.7/site-packages/sklearn/model_selection/tests/test_validation.pyt&   test_cross_val_score_sparse_fit_params  s    		#c          C   s<  d }  d } t  d |  d d d d d d d	 d
 d d d d  \ } } t |  | d |  } xt t g D]} t j d t  L } t | | | d t d |  d t j	 d d d  d | \ } } }	 Wd  QXt
 |  d k r t d | d j   n  t | j d  t |	 j d  t | t j	 d
 d d   t | j d d  t j	 d d d   t |	 j d d  t j	 d d d   t j d t  R } t | | | d t d | d |   d t j	 d d d  d | \ }
 } } Wd  QXt
 |  d k rt d | d j   n  t | |  t | |	  qj Wd  S(   Ni   i   R   R&  i   R'  t   n_redundanti    R   i   t   n_clusters_per_classR   R  R   R   RG   g?g      ?i
   R@   s   Unexpected warning: %ri   t   axisgffffff?(   i
   i   (   i
   i   (   RA   RE   R~   Rd   R  R  R!   R   Rq   t   linspaceRm   t   RuntimeErrort   messageR   RL   R   R   R@  RC   (   R   R   RP   R\   R   t   shuffle_traint   wRG   t   train_scorest   test_scorest   train_sizes2t   train_scores2t   test_scores2(    (    sL   lib/python2.7/site-packages/sklearn/model_selection/tests/test_validation.pyt   test_learning_curve  s>    c          C   s   t  d d d d d d d d d d	 d
 d d d  \ }  } t d  } t | |  d d  d d d t j d d d  \ } } } t | t j d	 d d   t | j d d  t j d d d   t | j d d  t j d d d   d  S(   NR   i   R&  i   R'  Rr  i    R   i   Rs  R   i   R\   R   i   RG   g?g      ?i
   Rt  gffffff?(	   RA   RE   R!   RH   Rq   Ru  R   R   R@  (   RP   R8  R   RG   Rz  R{  (    (    sL   lib/python2.7/site-packages/sklearn/model_selection/tests/test_validation.pyt    test_learning_curve_unsupervised  s    6c          C   s   t  d d d d d d d d d d	 d
 d d d  \ }  } t d  } t j } t   t _ z+ t | |  | d d d d \ } } } Wd  t j j   } t j j   | t _ Xd | k s t  d  S(   NR   i   R&  i   R'  Rr  i    R   i   Rs  R   i   R   i   t   verboses   [learning_curve](	   RA   RE   t   syst   stdoutt   StringIOR!   t   getvaluet   closeR   (   RP   R\   R   t
   old_stdoutRG   Rz  R{  t   out(    (    sL   lib/python2.7/site-packages/sklearn/model_selection/tests/test_validation.pyt   test_learning_curve_verbose+  s    	+
c          C   se   t  d d d d d d d d d d d	 d d
 d  \ }  } t d  } t t t | |  | d t d  S(   NR   i   R&  i   R'  Rr  i    R   Rs  R   t   exploit_incremental_learning(   RA   RE   R
   Ro   R!   Rd   (   RP   R\   R   (    (    sL   lib/python2.7/site-packages/sklearn/model_selection/tests/test_validation.pyt5   test_learning_curve_incremental_learning_not_possible>  s    c          C   s  t  d d d d d d d d d d	 d
 d d d  \ }  } t d  } x t t g D] } t | |  | d d d t d t j d d d  d | \ } } } t | t j d	 d d   t | j	 d d  t j d d d   t | j	 d d  t j d d d   qR Wd  S(   NR   i   R&  i   R'  Rr  i    R   i   Rs  R   i   R   i   R  RG   g?g      ?i
   R@   Rt  gffffff?(
   RA   RY   R~   Rd   R!   Rq   Ru  R   R   R@  (   RP   R\   R   Rx  RG   Rz  R{  (    (    sL   lib/python2.7/site-packages/sklearn/model_selection/tests/test_validation.pyt(   test_learning_curve_incremental_learningH  s    'c          C   s   t  d d d d d d d d d d	 d
 d d d  \ }  } t d  } t | |  d d  d d d t d t j d d d  \ } } } t | t j d	 d d   t | j	 d d  t j d d d   t | j	 d d  t j d d d   d  S(   NR   i   R&  i   R'  Rr  i    R   i   Rs  R   i   R\   R   i   R  RG   g?g      ?i
   Rt  gffffff?(
   RA   RY   R!   RH   Rd   Rq   Ru  R   R   R@  (   RP   R8  R   RG   Rz  R{  (    (    sL   lib/python2.7/site-packages/sklearn/model_selection/tests/test_validation.pyt5   test_learning_curve_incremental_learning_unsupervisedX  s    !s,   ignore:max_iter and tol parameters have beenc    
      C   s  t  d d d d d d d d d d	 d
 d d d  \ }  } t j d d d  } t d d d d  d t  } t | |  | d | d d d t \ } } } t | |  | d d d | d t \ } } }	 t | |  t	 | j
 d d  | j
 d d   t	 | j
 d d  |	 j
 d d   d  S(   NR   i   R&  i   R'  Rr  i    R   i   Rs  R   g?g      ?i   t   max_itert   tolR@   RG   R   i   R  Rt  (   RA   Rq   Ru  R4   RH   R~   R!   Rd   R   R   R@  (
   RP   R\   RG   R   t   train_sizes_inct   train_scores_inct   test_scores_inct   train_sizes_batcht   train_scores_batcht   test_scores_batch(    (    sL   lib/python2.7/site-packages/sklearn/model_selection/tests/test_validation.pyt<   test_learning_curve_batch_and_incremental_learning_are_equalh  s"    	c          C   s  t  d d d d d d d d d d	 d
 d d d  \ }  } t d  } t t t | |  | d d d d d g t t t | |  | d d d d d g t t t | |  | d d d d d g t t t | |  | d d d d d g t t t | |  | d d d d d g d  S(   NR   i   R&  i   R'  Rr  i    R   i   Rs  R   i   R   i   RG   g        g      ?g?g?i   (   RA   RE   R
   Ro   R!   (   RP   R\   R   (    (    sL   lib/python2.7/site-packages/sklearn/model_selection/tests/test_validation.pyt0   test_learning_curve_n_sample_range_out_of_bounds  s    c          C   s   t  d d d d d d d d d d	 d
 d d d  \ }  } t d	  } t t t | |  | d d d t j d d d  \ } } } t | d d	 g  d  S(   NR   i   R&  i   R'  Rr  i    R   i   Rs  R   R   RG   gQ?g      ?(   RA   RE   R   R\  R!   Rq   Ru  R   (   RP   R\   R   RG   R8  (    (    sL   lib/python2.7/site-packages/sklearn/model_selection/tests/test_validation.pyt1   test_learning_curve_remove_duplicate_sample_sizes  s    !c          C   s   t  d d d d d d d d d d	 d
 d d d  \ }  } t d  } t d d  } t | |  | d | d t j d d d  \ } } } t | t j d	 d d   t | j d d  t j d d d   t | j d d  t j d d d   d  S(   NR   i   R&  i   R'  Rr  i    R   i   Rs  R   i   R   i   R   RG   g?g      ?i
   Rt  gffffff?(	   RA   RE   R   R!   Rq   Ru  R   R   R@  (   RP   R\   R   R   RG   Rz  R{  (    (    sL   lib/python2.7/site-packages/sklearn/model_selection/tests/test_validation.pyt(   test_learning_curve_with_boolean_indices  s    3c          C   s  t  j d d g d d g d d g d d g d	 d
 g d d g d d g d d g d d g d d g d d g d	 d
 g d d g d d g d d g g  }  t  j d d d d d d d d d d d d d d d g  } t  j d d d d d d d d d d d d d d d g  } t d d d d  d t  } t d d  } t | |  | d | d d d t  j d d d  d | d t d d \ } } } t	 | j
 d  d  t  j d! d d" g   t	 | j
 d  d  t  j d" d# d# g   t t t | |  | d | d d d t  j d d d  d | d$ d% t | |  | d | d d d t  j d d d  d | d t d d d& t \ } }	 }
 t	 |	 j
 d  d  | j
 d  d   t	 |
 j
 d  d  | j
 d  d   d  S('   Ni   i   i   i   i   i   i   i   i   i   i   i   i   i   i   i   i   i   i	   i
   R  R  R@   R   R   R   RG   g333333?g      ?R.  R   Rt  g      ?gIq?g      ?R   R   R  (   Rq   R   R4   RH   R~   R   R!   Ru  Rd   R   R@  R
   Ro   (   RP   R\   R.  R   R   R  R  R  R  R  R  (    (    sL   lib/python2.7/site-packages/sklearn/model_selection/tests/test_validation.pyt    test_learning_curve_with_shuffle  s4    E6<<	-!-c          C   s   t  d d d d d d d d d d d	 d d
 d  \ }  } t j d d d  } t j d t  4 } t t   |  | d d d | d d \ } } Wd  QXt |  d k r t	 d | d j
   n  t | j d d  |  t | j d d  d |  d  S(   NR   i   R&  i   R'  Rr  i    R   Rs  R   i
   R  t
   param_nameR`   t   param_rangeR   s   Unexpected warning: %rRt  (   RA   Rq   Ru  R  R  Rd   R"   R_   Rm   Rv  Rw  R   R@  (   RP   R\   R  Ry  Rz  R{  (    (    sL   lib/python2.7/site-packages/sklearn/model_selection/tests/test_validation.pyt   test_validation_curve  s    c          C   s   t  d d d d d d d d d d d	 d d
 d  \ }  } t j d d d  } t t   |  | d d d | d d \ } } d  S(   NR   i   R&  i   R'  Rr  i    R   Rs  R   i
   R  R`   R  R   (   RA   Rq   Ru  R"   Ra   (   RP   R\   R  R8  (    (    sL   lib/python2.7/site-packages/sklearn/model_selection/tests/test_validation.pyt%   test_validation_curve_clone_estimator  s    c          C   sz  d }  d } t  d d d d  \ } } t t d d d d  | | d d	 d	 d
 d
 g d t d | d |   } t t j t j |  d d  d   f d    t t d d d d  | | d d	 d	 d
 d
 g d t d | d t	  } t t j t j |  d d  d   f d    t t d d d d  | | d d	 d	 d
 d
 g d t d |  } t t j
 |  t j
 |   d  S(   Nid   i   R   R   i    R   R   R   g?g?R   R   i   i   i   R@   (   i    i   i   i   (   i    i   i   i   (   RA   R"   R7   RC   R   Rq   t   vsplitt   hstackR   Rd   R   (   R   R   RP   R\   t   scores1t   scores2t   scores3(    (    sL   lib/python2.7/site-packages/sklearn/model_selection/tests/test_validation.pyt+   test_validation_curve_cv_splits_consistency  s$    %
%
c          C   s   t  j j d  }  t  j d  } |  j |  t | d  sC t  t t t  j | d  d   d | d <t t | d   t t t  j	 | d f  d   d  S(   Ni    id   i   (
   Rq   t   randomt   RandomStateR5  R@   R#   R   R   t   deleteR  (   t   rngRF  (    (    sL   lib/python2.7/site-packages/sklearn/model_selection/tests/test_validation.pyt   test_check_is_permutation  s    
c          C   s   t  d d d d d t d t d d  \ }  } t |   } t |  } t t d d	   } t | |  | d
 d } t | | | d
 d } | j   } t | |  d  S(   NR   i   t   n_labelsi   t   allow_unlabeledt   return_indicatorR   R   R   R   i
   (	   RB   R~   Rd   R   R?   R7   R   t   toarrayR   (   RP   R\   R   t   y_sparset   classifR_  t   preds_sparse(    (    sL   lib/python2.7/site-packages/sklearn/model_selection/tests/test_validation.pyt(   test_cross_val_predict_sparse_prediction  s    c      	   C   s  t    } | j | j } } t | | d d \ } } t t |   } t   } d d d g } xu| D]m} t |  | | d | } t t |  t |   t	 j
 t |  | g  }	 t |  |  }
 xL | j | |  D]8 \ } } |  j | | | |  |
 | |  |	 | <q Wt |  | | d | d | } t |	 |  t |  | | d d | d | } t | |  t |  | | d	 d | d | } t | |  t |  | | j d
  d | d | } t | |  qh Wd  S(   NR   i    Rc  RX  Rj  RW  R   i   i   t   str(   R'   R   R   R@   Rm   R   R   R   R   Rq   R   t   getattrR   RN   R   R   t   astype(   R   R   RP   R\   t   classesR   t   methodsRW  Rm  t   expected_predictionst   funcR   R   t   predictions_y1t   predictions_y2t   predictions_ystr(    (    sL   lib/python2.7/site-packages/sklearn/model_selection/tests/test_validation.pyt#   check_cross_val_predict_with_method-  s4    					c           C   s   t  t    d  S(   N(   R  R2   (    (    (    sL   lib/python2.7/site-packages/sklearn/model_selection/tests/test_validation.pyt"   test_cross_val_predict_with_methodT  s    s#   ignore: max_iter and tol parametersc          C   s#   t  d d d d  }  t |   d  S(   Nt   losst   logR   i   (   R3   R  (   R   (    (    sL   lib/python2.7/site-packages/sklearn/model_selection/tests/test_validation.pyt&   test_cross_val_predict_method_checking[  s    s    ignore: The default of the `iid`c          C   s9   t  t d d  i d d g d 6d d }  t |   d  S(   NR   i*   g?i   R   R   i   (   RD   R2   R  (   R   (    (    sL   lib/python2.7/site-packages/sklearn/model_selection/tests/test_validation.pyt/   test_gridsearchcv_cross_val_predict_with_methodd  s    	c         C   s   t  j t |  | g  } t | |  } x | j |  |  D] \ } }	 | j |  | | |  | |  |	  }
 | d k r t  j t |	  | f  } n- t  j t |	  | f t  j | j  j	  } |
 | d  d   | j
 f <| | |	 <q= W| S(   NRX  (   Rq   R   Rm   R  R   RN   R  t   finfoR   t   mint   classes_(   RP   R\   R   R  R   RW  R  R  R   R   t   expected_predictions_t   exp_pred_test(    (    sL   lib/python2.7/site-packages/sklearn/model_selection/tests/test_validation.pyt   get_expected_predictionso  s    c       	   C   s  t  j d  j d d  }  t  j g  t d  D] } | d ^ q.  } d } t d d  } t d d  } t   } d d	 d
 g } x| D]} t   }	 t |	 |  | d | d | }
 t	 |  | | | |	 |  } t
 | |
  t |	 |  | d | d | }
 t	 |  | | | |	 |  } t
 | |
  t t  j t d  d  d d } t |	 |  | d | d | }
 | j |  } t	 |  | | | |	 |  } t
 | |
  q Wd  S(   Ni   id   i   i
   R   i   i   Rc  RX  Rj  RW  R   R   i    (   Rq   R5  Rl   R   R   R   R:   R2   R   R  R   R@   RE  t   fit_transform(   RP   R[   R\   R  t   kfold3t   kfold4t   leR  RW  R   Rm  R  (    (    sL   lib/python2.7/site-packages/sklearn/model_selection/tests/test_validation.pyt#   test_cross_val_predict_class_subset  s4    ,				$	c       	      s<  t    }  |  j |  j } } t   } t j d d d t  } | j d  | j   t	 j
 | j d t	 j  t	 j
 | j d d d d d t	 j   zH t | | | d   f d	   t t t | | | d  f d
   Wd  d \    xG t d  D]9 } y t j | j  PWq t k
 r2t d  q Xq WXd  S(   Nt   modet   wbR  s   Hello world!!!!!R   RL   t   rR   c            s     S(   N(    (   R   RP   R\   (   RU   (    sL   lib/python2.7/site-packages/sklearn/model_selection/tests/test_validation.pyR     s    c            s     S(   N(    (   R   RP   R\   (   R   (    sL   lib/python2.7/site-packages/sklearn/model_selection/tests/test_validation.pyR     s    i   g      ?(    (   NN(   R'   R   R   Rf   t   tempfilet   NamedTemporaryFileR~   t   writeR  Rq   t   memmapt   nameRC  R   R
   Ro   RH   R   t   ost   unlinkt   WindowsErrorR   (   R   RP   R\   R   t   tfR8  (    (   RU   R   sL   lib/python2.7/site-packages/sklearn/model_selection/tests/test_validation.pyt   test_score_memmap  s&    		
'c             s   t  t  f g }  y- d d l m } m } |  j | | f  Wn t k
 rO n Xx |  D] \    t   } | j | j } }   |   |  } }   f d   }  f d   }	 t	 d | d |	  }
 t
 |
 | |  qW Wd  S(   Ni(   R   R   c            s   t  |     S(   N(   R   (   R[   (   R   (    sL   lib/python2.7/site-packages/sklearn/model_selection/tests/test_validation.pyR     s    c            s   t  |     S(   N(   R   (   R[   (   R   (    sL   lib/python2.7/site-packages/sklearn/model_selection/tests/test_validation.pyR     s    R   R   (   R   R   R   R   R   R   R'   R   R   R   R   (   R   R   R   R   RP   R\   R   R   R   R   R   (    (   R   R   sL   lib/python2.7/site-packages/sklearn/model_selection/tests/test_validation.pyt"   test_permutation_test_score_pandas  s    	c          C   s  t  t  j  }  t j d d  } t j d  } |  | d  t   d  d  d d  d  g	 } i d d 6} t t t	 | |  d } d | d | f } t
 t | t	 | |  d } t j t   t
 t | t	 |  Wd  QXi d	 d 6} t t d
 t	 | |  d } t t | t |  | d d d d t t | t |  | d d d d t t | t |  | | d d d d t t | t |  | | d t  j g d d d d t |  j   d  d  S(   Ni   i
   i	   i    R   s1   ValueError: Failing classifier failed as requiredsq   Estimator fit failed. The score on this train-test partition for these parameters will be set to %f. Details: 
%ss   From version 0.22, errors during fit will result in a cross validation score of NaN by default. Use error_score='raise' if you want an exception raised or error_score=np.nan to adopt the behavior from version 0.22.R   s%   Failing classifier failed as requireds   error_score must be the string 'raise' or a numeric value. (Hint: if using 'raise', please make sure that it has been spelled correctly.)R   i   s   unvalid-stringt	   parameterg        (   R   t   FAILING_PARAMETERRq   R5  R  RH   R   R   R   R$   R   t   pytestt   raisesRo   R   R   R   R   R!   R"   R   RU   (   t   failing_clfRP   R\   t   fit_and_score_argst   fit_and_score_kwargst   error_messageRa  (    (    sL   lib/python2.7/site-packages/sklearn/model_selection/tests/test_validation.pyt   test_fit_and_score  sD    

	(   i
   i   (   RX   t
   __future__R    R  R  R  R  t   timeR   R  t   numpyRq   t   scipy.sparseR   R   t   sklearn.exceptionsR   t)   sklearn.model_selection.tests.test_searchR   t   sklearn.utils.testingR   R   R   R	   R
   R   R   R   R   R   R   R   R   R   t   sklearn.utils.mockingR   R   t   sklearn.model_selectionR   R   R   R   R   R   R   R   R   R   R    R!   R"   t#   sklearn.model_selection._validationR#   R$   t   sklearn.datasetsR%   R&   R'   R(   t   sklearn.metricsR)   R*   R+   R,   R-   R.   R/   t   sklearn.metrics.scorerR0   t   sklearn.linear_modelR1   R2   R3   R4   R5   t   sklearn.neighborsR6   t   sklearn.svmR7   t   sklearn.clusterR8   t   sklearn.imputeR9   t   sklearn.preprocessingR:   t   sklearn.pipelineR;   t   sklearn.externals.six.movesR<   R  t   sklearn.baseR=   R>   t   sklearn.multiclassR?   t   sklearn.utilsR@   RA   RB   t$   sklearn.model_selection.tests.commonRC   RD   R  t	   NameErrorRH   RE   RY   R_   Ra   R  Rf   R  RP   R   R   R\   R   R  Rs   t   markt   filterwarningsR   R   R   R   R   R   R   R   R   R  R	  R  R  R   R%  R,  R=  RG  RH  RT  Rb  Rh  Ri  Rk  Ro  Rp  Rq  R  R  R  R  R  R  R  R  R  R  R  R  R  R  R  R  R  R  R  R  R  R  R  R  R  (    (    (    sL   lib/python2.7/site-packages/sklearn/model_selection/tests/test_validation.pyt   <module>   s:  
A--*G	*		-	1	!					2			!2!/!!!.
	$			
					$			"			'!$	!*