
    {Kg[                       d Z ddlZddlZddlZddlZddlZddlmZ ddlm	Z	 ddl
mZ ddlZddlZddlmZ ddlmZmZ ddlmZ dd	lmZmZmZmZmZmZ dd
lmZ ddlm Z  ddl!m"Z" ddl#m$Z$m%Z%m&Z&m'Z'm(Z( ddl)m*Z*m+Z+m,Z,m-Z-m.Z.m/Z/m0Z0m1Z1m2Z2 ddl3m4Z4 ddl5m6Z6m7Z7m8Z8m9Z9m:Z:m;Z;m<Z<m=Z=m>Z>m?Z?m@Z@mAZAmBZBmCZCmDZD ddlEmFZFmGZGmHZH ddlImJZJ ddlKmLZL ddlMmNZN ddlOmPZP ddlQmRZR ddlSmTZT ddlUmVZVmWZW ddlXmYZYmZZZ ddl[m\Z\m]Z]m^Z^m_Z_m`Z` ddlambZb ddlcmdZdmeZe ddlfmgZgmhZhmiZimjZj ddlkmlZlmmZm ddlnmoZo  G d  d!e      Zp G d" d#ep      Zq G d$ d%e      Zr G d& d'er      Zs G d( d)      Zt ej                  d*      Zv ej                  g d+      Zx ej                  g d,      Zy ej                  d-      Z{ej                  j                  d.el      d/        Z~d0 Zd1 Zd2 Zej                  j                  d3d4d5g      ej                  j                  d6em      d3efd7              Zd8 Zd9 Zd: Zej                  j                  d;      d<        Zd= Zd> Zej                  j                  d.el      d?        Zd@ ZdA ZdB Zej                  j                  d.el      dC        ZdD ZdE ZdF ZdG Zej                  j                  d.el      dH        ZdI ZdJ ZdK Zej                  j                  d.el      dL        Zej                  j                  d;      dM        ZdN ZdO Zej                  j                  d.el      dP        ZdQ ZdR ZdS ZdT ZdU ZdV ZdW ZdX ZdY ZdZ Zd[ Zd\ Zd] Zd^ Zd_ Zd` Zda Zdb Zej                  j                  d6em      dc        Zdd Zde Zdf Zdg Zdh Zdi Zdj Zdk Zdl Z G dm dne      Zdo Zdp Zdq Zdr Zds Zdt Zej                  j                  d;      du        Zdv Zdw Z G dx dye      Zej                  j                  dzej                  dg      d{        Zej                  j                  dzej                  dg      d|        Zd} Zej                  j                  d~      ej                  j                  dzej                  ddg      d               Zej                  j                  d~      ej                  j                  dzej                  ddg      ej                  j                  dd5d4g      ej                  j                  dd4d5g      d                             Zd Zej                  j                  dd4eddddfd5 e4eed      ddddfd4 e4eed      ddddfg      d        Zd Zd Zd Zd Zd Zd Zej                  j                  d      ej                  j                  deAe@e?g      d               Zej                  j                  d      ej                  j                  deAe@e?g      d               Zej                  j                  d      ej                  j                  deAe@e?g      d               Zy)zTest the validation module    N)partial)StringIO)sleep)issparse)BaseEstimatorclone)KMeans)load_diabetesload_digits	load_irismake_classificationmake_multilabel_classificationmake_regression)RandomForestClassifier)FitFailedWarning)SimpleImputer)LogisticRegressionPassiveAggressiveClassifierRidgeRidgeClassifierSGDClassifier)	accuracy_scorecheck_scoringconfusion_matrixexplained_variance_scoremake_scorermean_squared_errorprecision_recall_fscore_supportprecision_scorer2_score)_MultimetricScorer)GridSearchCV
GroupKFoldGroupShuffleSplitKFoldLeaveOneGroupOutLeaveOneOutLeavePGroupsOutShuffleSplitStratifiedKFoldcross_val_predictcross_val_scorecross_validatelearning_curvepermutation_test_scorevalidation_curve)_check_is_permutation_fit_and_score_score)OneTimeSplitter)FailingClassifier)OneVsRestClassifier)KNeighborsClassifier)MLPRegressor)Pipeline)LabelEncoderscale)SVC	LinearSVC)ConsumingClassifierConsumingScorerConsumingSplitter	_Registrycheck_recorded_metadata)shuffle)CheckingClassifierMockDataFrame)assert_allcloseassert_almost_equalassert_array_almost_equalassert_array_equal)COO_CONTAINERSCSR_CONTAINERS)_num_samplesc                   2    e Zd ZdZd ZddZd Zd	dZd Zy)
MockImprovingEstimatorz+Dummy classifier to test the learning curvec                 .    || _         d| _        d | _        y Nr   )n_max_train_sizestrain_sizesX_subset)selfrQ   s     q/home/alanp/www/video.onchill/myenv/lib/python3.12/site-packages/sklearn/model_selection/tests/test_validation.py__init__zMockImprovingEstimator.__init__d   s    !2    Nc                 <    || _         |j                  d   | _        | S rP   rS   shaperR   rT   rS   y_subsets      rU   fitzMockImprovingEstimator.fiti        #>>!,rW   c                     t         NNotImplementedErrorrT   Xs     rU   predictzMockImprovingEstimator.predictn       !!rW   c                     | j                  |      r%dt        | j                        | j                  z  z
  S t        | j                        | j                  z  S )Ng       @)_is_training_datafloatrR   rQ   rT   rd   Ys      rU   scorezMockImprovingEstimator.scoreq   sM    !!!$t//043I3IIII))*T-C-CCCrW   c                     || j                   u S r`   rS   rc   s     rU   rh   z(MockImprovingEstimator._is_training_datax       DMM!!rW   r`   NN	__name__
__module____qualname____doc__rV   r]   re   rl   rh    rW   rU   rN   rN   a   s    5

"D"rW   rN   c                   2     e Zd ZdZd fd	Zd ZddZ xZS )!MockIncrementalImprovingEstimatorz*Dummy classifier that provides partial_fitc                 @    t         |   |       d | _        || _        y r`   )superrV   xexpected_fit_params)rT   rQ   r|   	__class__s      rU   rV   z*MockIncrementalImprovingEstimator.__init__   s     *+#6 rW   c                     | j                   |v S r`   r{   rc   s     rU   rh   z3MockIncrementalImprovingEstimator._is_training_data   s    vv{rW   c                    | xj                   |j                  d   z  c_         |d   | _        | j                  rt	        | j                        t	        |      z
  }|rt        dt        |       d      |j                         D ]R  \  }}|| j                  v st        |      t        |      k7  s-t        d| dt        |       dt        |       d       y y )Nr   zExpected fit parameter(s) z
 not seen.zFit parameter z has length z; expected .)	rR   rZ   r{   r|   setAssertionErrorlistitemsrL   )rT   rd   yparamsmissingkeyvalues          rU   partial_fitz-MockIncrementalImprovingEstimator.partial_fit   s    AGGAJ&1##$223c&kAG$0gzJ  %lln
U$222|8!!_8% )(\,u:M9N%l1o%6a9 	 - $rW   r`   )rr   rs   rt   ru   rV   rh   r   __classcell__r}   s   @rU   rx   rx   |   s    47
rW   rx   c                   2    e Zd ZdZddZd Zd Zd	dZd Zy)
MockEstimatorWithParameterz-Dummy classifier to test the validation curvec                      d | _         || _        y r`   )rS   param)rT   r   s     rU   rV   z#MockEstimatorWithParameter.__init__   s    
rW   c                 <    || _         |j                  d   | _        | S rP   rY   r[   s      rU   r]   zMockEstimatorWithParameter.fit   r^   rW   c                     t         r`   ra   rc   s     rU   re   z"MockEstimatorWithParameter.predict   rf   rW   Nc                 Z    | j                  |      r| j                  S d| j                  z
  S )N   )rh   r   )rT   rd   r   s      rU   rl   z MockEstimatorWithParameter.score   s&    !33A6tzzJA

NJrW   c                     || j                   u S r`   rn   rc   s     rU   rh   z,MockEstimatorWithParameter._is_training_data   ro   rW   )      ?rp   rq   rv   rW   rU   r   r      s    7
"K"rW   r   c                   (     e Zd ZdZ fdZd Z xZS )%MockEstimatorWithSingleFitCallAllowedz<Dummy classifier that disallows repeated calls of fit methodc                 X    t        | d      rJ d       d| _        t        |   ||      S )Nfit_called_zfit is called the second timeT)hasattrr   rz   r]   )rT   rS   r\   r}   s      rU   r]   z)MockEstimatorWithSingleFitCallAllowed.fit   s2    4/P1PP/w{8X..rW   c                     t         r`   ra   rc   s     rU   re   z-MockEstimatorWithSingleFitCallAllowed.predict   rf   rW   )rr   rs   rt   ru   r]   re   r   r   s   @rU   r   r      s    F/
"rW   r   c                   N    e Zd ZdZd	dZ	 	 	 	 	 	 	 	 	 d
dZd Zd ZddZddZ	y)MockClassifierz-Dummy classifier to test the cross-validationc                      || _         || _        y r`   aallow_nd)rT   r   r   s      rU   rV   zMockClassifier.__init__   s     rW   Nc           	         || _         || _        |	| _        |
 |
|        | j                  r|j	                  t        |      d      }|j                  dk\  r| j                  st        d      |P|j                  d   |j                  d   k(  s1J dj                  |j                  d   |j                  d                |v|j                  d   t        t        j                  t                    k(  sDJ dj                  |j                  d   t        t        j                  t                                 |Rd}|j                  d   |j                  d   k(  s1J |j                  |j                  d   |j                  d                |td}|j                  t        j                  k(  sUJ |j                  |j                  d   |j                  d	   t        j                  d   t        j                  d	                | S )
zThe dummy arguments are to test that this fit function can
        accept non-array arguments through cross-validation, such as:
            - int
            - str (this is actually array-like)
            - object
            - function
           zX cannot be dr   zKMockClassifier extra fit_param sample_weight.shape[0] is {0}, should be {1}zIMockClassifier extra fit_param class_prior.shape[0] is {0}, should be {1}zRMockClassifier extra fit_param sparse_sample_weight.shape[0] is {0}, should be {1}zUMockClassifier extra fit_param sparse_param.shape is ({0}, {1}), should be ({2}, {3})r   )	dummy_int	dummy_str	dummy_objr   reshapelenndim
ValueErrorrZ   formatnpuniquer   P)rT   rd   rk   sample_weightclass_priorsparse_sample_weightsparse_paramr   r   r   callbackfmts               rU   r]   zMockClassifier.fit   s   ( #""TN==		#a&"%A66Q;t}}_--$ &&q)QWWQZ7 ??Ev!''*AGGAJ@7 "$$Q'3ryy|+<< ))/0A0A!0Dc"))TU,FW)X<  +2  (--a0AGGAJ> 

$**1-qwwqzA > #6   %%0 #**""1%""1%

	3 0 rW   c                 b    | j                   r|j                  t        |      d      }|d d df   S )Nr   r   )r   r   r   rT   Ts     rU   re   zMockClassifier.predict  s*    ==		#a&"%AAwrW   c                     |S r`   rv   r   s     rU   predict_probazMockClassifier.predict_proba  s    rW   c                 L    ddt        j                  | j                        z   z  S )N      ?r   )r   absr   rj   s      rU   rl   zMockClassifier.score
  s    a"&&.())rW   c                 4    | j                   | j                  dS )Nr   r   )rT   deeps     rU   
get_paramszMockClassifier.get_params  s    VV77rW   )r   F)	NNNNNNNNNrp   )F)
rr   rs   rt   ru   rV   r]   re   r   rl   r   rv   rW   rU   r   r      sA    7! !=~
*8rW   r   )
      )
r   r   r   r   r   r   r   r      r   )
r   r   r   r   r   r   r   r   r   r      coo_containerc                    t               } | t              }t        dd      D ]  }||_        t	        |t        t
              }t        ||j                  t        t
                     t        j                  t
        t
        d d d   g      }t	        |||      }t        ||j                  ||             t	        ||t
              }t        ||j                  |t
                     t	        |||      }t        ||j                  ||              d }t        |      }t	        |t        j                         t
        j                         d      }t        |      }t	        |t        t
        j                         d      }t        d d d d t        j                  f   }t        d	
      }t	        ||t
              }t        d
      }t        j                  t              5  t	        ||t
        d       d d d        y # 1 sw Y   y xY w)Nir   r   c                 "    t        | t              S r`   
isinstancer   r   s    rU   <lambda>z&test_cross_val_score.<locals>.<lambda>3      :a.rW   check_Xr   cvcheck_yT)r   Fraiseerror_score)r   rd   ranger   r,   y2rI   rl   r   column_stackrD   tolistnewaxispytestraisesr   )r   clfX_sparser   scoresmultioutput_y
list_checkX_3ds           rU   test_cross_val_scorer     s   

CQH3^ a,6399Q#34 R"X7 h>6399X}#EF h36399Xr#:; !h>6399X}#EF! & /J
Z
0CS!((*biika@F
Z
0CS!RYY[Q7F Q2::D
$
'CS$+F
%
(C	z	"T27; 
#	"	"s   #H  H	c                  z    t        d      \  } }t        d      }t        |dddgi      }t        || |d	
       y )NT
return_X_yauto)gammaCr   r   )
param_gridr   )n_jobs)r   r<   r"   r-   )rd   r   r   grids       rU   test_cross_validate_many_jobsr   D  s>     %DAq
F
Cq"g7D4Aa(rW   c            
         t        d      \  } }t               }d}t        j                  t        |      5  t        || |t        t              t        t              f       d d d        t        j                  t        |      5  t        || |t        t              f       d d d        t        j                  t        |dz         5  t        || |d       d d d        t        j                  t        |dz         5  t        || |d	       d d d        t        j                  t        |      5  t        || |t        t              gg       d d d        t        j                  t        d
      5  t        || |t                      d d d        t        t              }dt        j                   d}t        j                  t        |      5  t        || ||       d d d        t        j                  t        |      5  t        || |d|i       d d d        y # 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   cxY w# 1 sw Y   :xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   y xY w)Nr   random_statez.*must be unique strings.*matchscoringzEmpty list.*rv   zDuplicate.*)f1_micror   zAn empty dict[Scoring failed. The score on this train-test partition for these parameters will be set to z. Details: 
foo)r   r   r   r   r   r-   r   r   r   dictr   r   nanwarnsUserWarning)rd   r   	estimatorerror_message_regexpmulticlass_scorerwarning_messages         rU   )test_cross_validate_invalid_scoring_paramr  N  s   A.DAq I 8 
z)=	> 1;~3NO		
 
? 
z)=	>y!Q_1M0OP 
? 
z)=)N	Oy!Q3 
P 
z)=)M	Ny!Q0HI 
O 
z)=	>y!Q+o2N1O0PQ 
? 
z	9y!Q9 
: $$CD
	99; A	  
k	9y!Q0AB 
: 
k	9y!Q8I0JK 
:	9O 
?	> 
?	> 
P	O 
O	N 
?	> 
:	9 
:	9 
:	9s_   ,H!H.
H; I3I5I"I.I:!H+.H8;III"I+.I7:Jc                      t        d      \  } }t        dt               fdt               fg      }t	        || |d      }|d   }t        |t              sJ t        d |D              sJ y )NTr   imputer
classifier)return_estimatorr   c              3   <   K   | ]  }t        |t                y wr`   )r   r9   ).0r   s     rU   	<genexpr>z7test_cross_validate_nested_estimator.<locals>.<genexpr>  s     K
9z)X.
s   )r   r9   r   r   r-   r   r   all)rd   r   pipelineresults
estimatorss        rU   $test_cross_validate_nested_estimatorr    sv     $'FQ(>+,	
H Xq!dCG%Jj$'''K
KKKKrW   
use_sparseFTcsr_containerc           
         t               }t        dd      \  }}t        d      }t        dd      \  }}t	        dd      }| r ||      } ||      }|||f|||ffD ]  \  }	}
}t        |d      }t        |d	      }g }g }g }g }g }|j                  |	|
      D ]  \  }}t        |      j                  |	|   |
|         }|j                   |||	|   |
|                |j                   |||	|   |
|                |j                   |||	|   |
|                |j                   |||	|   |
|                |j                  |        t        j                  |      }t        j                  |      }t        j                  |      }t        j                  |      }t        j                  |      }|||||f}t        ||	|
||       t        ||	|
||        y )
N   r   	n_samplesr   r   linearkernelr   neg_mean_squared_errorr   r2)r%   r   r   r   r<   r   splitr   r]   appendr   array"check_cross_validate_single_metric!check_cross_validate_multi_metric)r  r  r   X_regy_regregX_clfy_clfr   rd   r   est
mse_scorer	r2_scorertrain_mse_scorestest_mse_scorestrain_r2_scorestest_r2_scoresfitted_estimatorstraintestr   s                         rU   test_cross_validater-    s    
B #Ra@LE5
Q
C '!DLE5
XA
.Ce$e$eS)E5#+>?	1c"30HI
!#t4	88Aq>KE4*..51U84C##JsAeHah$GH""9S!E(AeH#EF"":c1T7AdG#DE!!)C4!D'"BC$$S) * 88$45((?3((?3.1HH%67 
 	+31fbA)#q!VR@I @rW   c           	      H   |\  }}}}}	dD ]  \  }
}|
r!t        | ||dd|      }t        |d   |       nt        | ||dd|      }t        |t              sJ t	        |      |k(  sJ t        |d   |       |
r#t        | ||dgd|      }t        |d	   |d       nt        | ||dgd|      }t        |t              sJ t	        |      |k(  sJ t        |d
   |        t        | ||dd|      }t        |d         D ]  \  }}|j                  j                         }t        |      r|j                         }|	|   j                  j                         }t        |      r|j                         }t        ||       t        |j                  |	|   j                          y )N))Tr   )Fr   r  Tr   return_train_scorer   train_scoreF
test_scorer  train_r2test_r2)r   r  r   r   )r-   rH   r   r   r   	enumeratecoef_copyr   toarrayrG   
intercept_)r   rd   r   r   r   r&  r'  r(  r)  r*  r0  dict_lenmse_scores_dictr2_scores_dictkr#  est_coeffitted_est_coefs                     rU   r  r    s    	 )@$H,0#'O &om&DFVW,0#(O /4000?#x///!/,"?Q +QD6drN &nZ&@/SWX+QD6eN .$///>"h...!.";^LM )@R %Q3drO OK89399>>#H'')H+A.4499;O$-557OHo6CNN,=a,@,K,KL :rW   c           
         |\  }}}}}	d }
dt        t              dd|
f}h d}|j                  ddh      }dD ]  }|D ]  }|r0t        | |||d|	      }t	        |d   |       t	        |d   |       nt        | |||d
|	      }t        |t              sJ t        |j                               |r|n|k(  sJ t	        |d   |       t	        |d   |       t        |d   t        j                        sJ t        |d   t        j                        sJ t        |d   t        j                        sJ t        |d   t        j                        sJ t        j                  |d   dk\        sJ t        j                  |d   dk        sJ t        j                  |d   dk\        sJ t        j                  |d   dk        rJ   y )Nc                 X    | j                  |      }t        ||      t        ||       dS )Nr  r  )re   r    r   )r   rd   r   y_preds       rU   custom_scorerz8check_cross_validate_multi_metric.<locals>.custom_scorer  s0    Q1f%'9!V'D&D
 	
rW   rB  r  >   r4  fit_time
score_timetest_neg_mean_squared_errorr3  train_neg_mean_squared_error)TFTr/  Fr4  rG  rE  rF  r   r   )r   r    unionr-   rH   r   r   r   keysr   ndarrayr	  )r   rd   r   r   r   r&  r'  r(  r)  r*  rD  all_scoringkeys_sans_trainkeys_with_trainr0  r   
cv_resultss                    rU   r  r    s    	
 	)h'&>	
 	KO &++	34O ,"G!+Aw4B
 **Z*@/R)=>@P ,Aw5R
 j$///z()#5?   &j&;^L%89?
 j3RZZ@@@j)FGTTTj4bjjAAAj6

CCC 66*Z0A566666*Z02566666*\2a788866*\2R7888C # ,rW   c                     t        ddd      \  } }t        d      }t               t        d      t	               t               g}d}|D ]h  }t        j                  t        |      5  t        || ||	       d d d        t        j                  t        |      5  t        || ||	       d d d        j y # 1 sw Y   ?xY w# 1 sw Y   xY w)
N   r   r   )r  	n_classesr   r  r  z*The 'groups' parameter should not be None.r   )r   rd   r   r   )r   r<   r&   r(   r#   r$   r   r   r   r,   r+   )rd   r   r   	group_cvserror_messager   s         rU   #test_cross_val_score_predict_groupsrV  Y  s     qqIDAq
X
C 		I AM]]:];cQ!; <]]:];qA"= <; ;;;;s   'B5C5B>	C
	z(ignore: Using or importing the ABCs fromc                    	 t         t         fg} 	 ddlm}m} | j	                  ||f       | D ]C  \  	 t               	t              }}fd}	fd}t        ||      }t        |||d       E y # t
        $ r Y Tw xY w)Nr   	DataFrameSeriesc                     t        |       S r`   r   r{   InputFeatureTypes    rU   r   z-test_cross_val_score_pandas.<locals>.<lambda>}      Z+;<rW   c                     t        |       S r`   r\  r{   
TargetTypes    rU   r   z-test_cross_val_score_pandas.<locals>.<lambda>~      Az!:rW   r   r   r   r   )
rE   pandasrY  rZ  r  ImportErrorrd   r   rD   r,   
typesrY  rZ  X_dfy_sercheck_dfcheck_seriesr   r^  rb  s
           @@rU   test_cross_val_score_pandasrm  o  s     ]+,E,fi() ).$
$ 'q):b>e<: <HT5Q/ ).     A5 5	B Bc                     t        d      } t               }|j                  |j                  }}t	        d      }t        | |||      }t	        d      }g }|j                  ||      D ]j  \  }}t        j                  t        |      t              }	t        j                  t        |      t              }
d|	|<   d|
|<   |j                  ||f       l t        | |||      }t        ||       y )Nr  rS  r   r   dtyper   )r<   r   datatargetr%   r,   r  r   zerosr   boolr  rI   )svmirisrd   r   kfoldscores_indicescv_masksr+  r,  
mask_train	mask_testscores_maskss               rU   test_cross_val_score_maskr~    s    
X
C;D99dkkqA!HE$S!Q59N!HEH{{1a(tXXc!fD1
HHSV40	
5	$& ) #31:L~|4rW   c                  p   t        d      } t               }|j                  |j                  }}t	        j
                  ||j                        }t        | ||      }t        d      } t        | ||      }t        ||       t        d       } t        | ||      }t        ||       t        d      } t        j                  t              5  t        | ||       d d d        t        j                  t              5  t        | |j                         |       d d d        y # 1 sw Y   GxY w# 1 sw Y   y xY w)NprecomputedrS  r  c                 B    t        j                  | |j                        S r`   )r   dotr   )r{   r   s     rU   r   z2test_cross_val_score_precomputed.<locals>.<lambda>  s    "&&ACC.rW   )r<   r   rr  rs  r   r  r   r,   rH   r   r   r   r   )rv  rw  rd   r   linear_kernelscore_precomputedscore_linearscore_callables           rU    test_cross_val_score_precomputedr    s    
]
#C;D99dkkqAFF1accNM']A>
X
C"31-L/> 0
1C$S!Q/N/@ ]
#C	z	"Q" 
#
 
z	"]113Q7 
#	" 
#	"
 
#	"s   D ;D, D),D5c           	        	
 t               }t        j                  d   }t        t	        j
                  t                    } | t	        j                  dg      t	        j                  dg      t	        j                  dg      ffd      } | t	        j                  d            }dd
t               		
fd}t	        j                  |      t	        j                  |d	|z        ||
	|d
}t        |t        t        |       y )Nr   r   )r   r   )rZ   r   *   42c                 l    | j                   k(  sJ | j                  k(  sJ | j                  k(  sJ y r`   )r   r   r   )r   	DUMMY_INT	DUMMY_OBJ	DUMMY_STRs    rU   assert_fit_paramsz:test_cross_val_score_fit_params.<locals>.assert_fit_params  s;     }}	)))}}	)))}}	)))rW   r   )r   r   r   r   r   r   r   r   )r   )r   rd   rZ   r   r   r   r   r  eyeobjectonesfullr,   )r   r   r  rR  W_sparseP_sparser  
fit_paramsr  r  r  s           @@@rU   test_cross_val_score_fit_paramsr    s    

C
IBIIaL!I	1#1#!67wH RVVAY'HIII* +wwy#	/: ( %	J CAj1rW   c                      t               } g fd}t        j                  d      5  t        |      }t	        | t
        t        |d      }d d d        t        g d       t              dk(  sJ y # 1 sw Y   (xY w)Nc                 ,    j                  | |f       y)Nr   )r  )y_test	y_predict_score_func_argss     rU   
score_funcz3test_cross_val_score_score_func.<locals>.score_func  s     34rW   Trecordr   )r   r   )r   r   r   )	r   warningscatch_warningsr   r,   rd   r   rI   r   )r   r  r   rl   r  s       @rU   test_cross_val_score_score_funcr    sp    

C 
	 	 	-j)Q7qA 
. uo. A%%% 
.	-s   $A44A=c                  X   t               } t        d      }t        || j                  | j                        }t        |g dd       t        || j                  | j                  d      }t        |g dd       t        || j                  | j                  d      }t        |g dd       y )Nr  rS  )
ףp=
?r   r  r  r   r   accuracyr   f1_weighted)r   r<   r,   rr  rs  rH   )rw  r   r   	zo_scores	f1_scoress        rU   3test_cross_val_score_with_score_func_classificationr    s    ;D
X
C S$))T[[9Ff&BAF  TYYZPIi)EqI  TYY]SIi)EqIrW   c                  p   t        dddd      \  } }t               }t        || |      }t        |g dd       t        || |d	      }t        |g dd       t        || |d
	      }t	        j
                  g d      }t        ||d       t        t              }t        || ||	      }t        |g dd       y )Nr  rQ  r   r   )r  
n_featuresn_informativer   )gGz?r  r  gGz?gq=
ףp?r   r  r   r  )g(\؇gzGIgGz&qg\(qg)\G)r   r   r,   rH   r   r  r   r   )	rd   r   r   r   	r2_scoresneg_mse_scoresexpected_neg_mser   	ev_scoress	            rU   /test_cross_val_score_with_score_func_regressionr    s    RBaVWXDAq
'C S!Q'Ff&DaH  Q48Ii)GK %S!Q8PQNxx NOn.>B 23GQ7;Ii)GKrW   c                 4   t               }|j                  } | |      }|j                  }t        d      }t	        d      }t        |||d|d      \  }}}	|dkD  sJ t        |	dd	       t        |||d|dt        j                  |j                        d
      \  }
}}|
|k(  sJ ||	k(  sJ t        d      }t	        d      }t        |||d|dt        j                  |j                        d
      \  }
}}|
|k(  sJ ||	k(  sJ d }t        |      }t        |||d||d
      \  }}}	t        |dd       t        |	dd       t        j                  t        j                  t        |            d      }t        |||d|d      \  }}}	|dk  sJ |	dkD  sJ y )Nr  rS  r   r  r  )n_permutationsr   r   g?        r   r   )r  r   r   groupsr   c                 p    | |k(  j                         | |k7  j                         z
  | j                  d   z  S rP   )sumrZ   )y_truerC  s     rU   custom_scorez,test_permutation_score.<locals>.custom_score>  s7    6!&&(Ff,<+A+A+CCv||TUVVrW   d   )r  r   r   r   g(\?g{Gz?r   r   皙?)r   rr  rs  r<   r*   r/   rG   r   r  sizer   modaranger   )r   rw  rd   r   r   rv  r   rl   r   pvaluescore_group_pvalue_group
svm_sparse	cv_sparser  scorers                    rU   test_permutation_scorer    s   ;D		AQHA
X
C		B2Q"ZE66 3;;Q'#9		wwqvv	$ KL %6!!! H%J"I#9	wwqvv	$ KL %6!!!W &F-Q#v"1E1f tQ'a( 	ryyQ !$A2Q"ZE66 3;;C<<rW   c                  j   t        j                  dt         j                        j                  dd      } t         j                  | dd d f<   t        j
                  ddg| j                  d   dz        }t        dt        d	t         j                  
      fdt               fg      }t        || |       y N   rp  r   r   r   r   r   r  mean)strategymissing_valuesr  )r   r  float64r   r   repeatrZ   r9   r   r   r/   rd   r   ps      rU   &test_permutation_test_score_allow_nansr  S  s    
		#RZZ(00R8AffAadG
		1a&!''!*q.)AvbffMN>+,	
	A 1a#rW   c            
         t        j                  d      j                  dd      } t        j                  dgdz  dgdz  z         }t	        d      }d}t        j                  t        |	      5  t        || |       d d d        d
}t        j                  t        |	      5  t        || |dt        j                  d      i       d d d        t        || |dt        j                  d      i       y # 1 sw Y   xxY w# 1 sw Y   :xY w)Nr  r   r   r   r   Texpected_sample_weight#Expected sample_weight to be passedr   /sample_weight.shape == \(1,\), expected \(8,\)!r   )r  )r   r  r   r  rD   r   r   r   r/   r   r  rd   r   r   err_msgs       rU   &test_permutation_test_score_fit_paramsr  a  s    
		#r2&A
!qA37"#A
D
9C4G	~W	5sAq) 
6 AG	z	1sAqorwwqz5RS 
231/2772;1OP 
6	5 
2	1s   /C4"%D 4C= D	c                  j   t        j                  dt         j                        j                  dd      } t         j                  | dd d f<   t        j
                  ddg| j                  d   dz        }t        dt        d	t         j                  
      fdt               fg      }t        || |       y r  )r   r  r  r   r   r  rZ   r9   r   r   r,   r  s      rU   test_cross_val_score_allow_nansr  p  s    
		#RZZ(00R8AffAadG
		1a&!''!*q.)AvbffMN>+,	
	A Aq!rW   c                     t        j                  ddgddgddgddgddgddgddgddgddgddgg
      } t        j                  ddgddgddgddgddgddgddgddgddgddgg
      }t        d	      }t        t        d
      }t        t        d      }t        t        d      }t        || ||      }t        || ||      }t        || ||      }t        |g d       t        |g d       t        |g d       y )Nr   r   r   r   r   r   )n_neighborsmicro)averagemacrosamplesr   )r   r         ?r   gUUUUUU?)r   r   r  r         ?)r   r  r7   r   r   r,   rG   )	rd   r   r   scoring_microscoring_macroscoring_samplesscore_microscore_macroscore_sampless	            rU   test_cross_val_score_multilabelr  ~  s/   
GFFFGGFHHG	
	A 	
Q!Q!Q!Q!Q!Q!Q!Q!QRSUVQWX	A 1
-CAMAM!/9EO!#q!]CK!#q!]CK#CAGM%DE%DE'FGrW   c           
      f   t        d      \  }}t               }t               }t        j                  |      }|j                  ||      D ]4  \  }}|j                  ||   ||          |j                  ||         ||<   6 t        ||||      }t        ||       t        |||      }t        |      t        |      k(  sJ t               }t        ||||      }t        |      t        |      k(  sJ |j                         }	|	|	t        j                  |	      kD  z  }	 | |	      }	t        ||	|      }t        t        |      t        |             t        t        d      |      }t        |      t        |      k(  sJ  G d d      }
t        j                   t"              5  t        ||| |
              d d d        t%        d      \  }}d}t        j&                  t(        |	      5  t        t+        d
      ||dt        d             d d d        y # 1 sw Y   bxY w# 1 sw Y   y xY w)NTr   r   r   )n_initc                       e Zd ZddZy)%test_cross_val_predict.<locals>.BadCVNc              3      K   t        d      D ]2  }t        j                  g d      t        j                  g d      f 4 y w)Nr   )r   r   r   r   )r   r            )r   r   r  )rT   rd   r   r  is        rU   r  z+test_cross_val_predict.<locals>.BadCV.split  s1     1Xhh|,bhh.GGG s   AArp   )rr   rs   rt   r  rv   rW   rU   BadCVr    s    	HrW   r  zNumber of classes in training fold \(2\) does not match total number of classes \(3\). Results may not be appropriate for your use case.r   	liblinearsolverr   r   methodr   )r
   r%   r   r   
zeros_liker  r]   re   r+   rH   r   r'   r7  medianr	   r   r   r   r   r   RuntimeWarningr   )r   rd   r   r   r#  preds2r+  r,  predsXspr  r   s               rU   test_cross_val_predictr    s   D)DAq	B
'C ]]1Fxx1~t%!E(#{{1T7+t & c1aB/EeV,c1a(Eu:Q	Bc1aB/Eu:Q
&&(C33C

Cc3*Ec%j#a&1fF3Q7Eu:QH H
 
z	"#q!0 
# %DAq	< 
 
nO	<k2"Qx	
 
=	< 
#	" 
=	<s   %H.$H'H$'H0c            
         t        ddd      \  } }t        t        d      | |d      }|j                  d	k(  sJ t	        d
      \  } }t        t        d      | |d      }|j                  dk(  sJ | d d } |d d }d}t        j                  t        |      5  t        t               | |dt        d             d d d        t        d
      \  } }t        dd      }t        || |d      }|j                  dk(  sJ t        j                  |      }| |   ||   }} d}t        j                  t        |      5  t        || |t        d      d       d d d        y # 1 sw Y   xY w# 1 sw Y   y xY w)Nr   2   r   rR  r  r   r  r  decision_functionr  )r  Tr      r   r  zOnly 1 class/es in training fold, but 2 in overall dataset. This is not supported for decision_function with imbalanced folds. To fix this, use a cross-validation technique resulting in properly stratified foldsr   r  r  ovo)r  decision_function_shape)i  -   zOutput shape \(599L?, 21L?\) of decision_function does not match number of classes \(7\) in fold. Irregular decision_function .*r   n_splitsr   r  )r   r+   r   rZ   r   r   r   r   r   r%   r   r<   r   argsort)rd   r   r   rU  r#  indr   s          rU   .test_cross_val_predict_decision_function_shaper    sv   bqIDAq+.1=PE ;;%%DAq+.1=PE ;;(""" 	
$3A	$3A	1  
z	7q!,?E!H	
 
8
 $'DAq
Xu
=Cc1a0CDE;;*$$$
**Q-CS61S6qA	  
z)=	>#q!q(9BUV 
?	>' 
8	7& 
?	>s   "E">E."E+.E7c                      t        ddd      \  } }t        t        d      | |d      }|j                  d	k(  sJ t	        d
      \  } }t        t        d      | |d      }|j                  dk(  sJ y )Nr   r  r   r  r  r  r   r  r  r   Tr   r  r   r+   r   rZ   r   rd   r   r   s      rU   *test_cross_val_predict_predict_proba_shaper  	  sz    bqIDAq+.1_E ;;'!!!%DAq+.1_E ;;("""rW   c                      t        ddd      \  } }t        t        d      | |d      }|j                  d	k(  sJ t	        d
      \  } }t        t        d      | |d      }|j                  dk(  sJ y )Nr   r  r   r  r  r  predict_log_probar  r  Tr   r  r  r  s      rU   .test_cross_val_predict_predict_log_proba_shaper    s|    bqIDAq+.1=PE ;;'!!!%DAq+.1=PE ;;("""rW   c                    t               }|j                  |j                  }} | |      }t        j                  ||d d d   g      }t        dd      }t        |||      }|j                  dk(  sJ t        |||      }|j                  dk(  sJ t        |||      }t        |j                  d       t        |||      }t        |j                  d       d }t        |      }t        ||j                         |j                               }t        |	      }t        |||j                               }t        t        d
      |j                         |j                         d      }t        t        d
      ||j                         d      }|d d d d t        j                  f   }	d }
t        |
      }t        ||	|      }t        |j                  d       y )Nr   Fr   )fit_interceptr   )r	  )r	  r   c                 "    t        | t              S r`   r   r   s    rU   r   z4test_cross_val_predict_input_types.<locals>.<lambda>B  r   rW   r   r   r  r  r  r  c                      | j                   dk(  S )Nr   )r   r   s    rU   r   z4test_cross_val_predict_input_types.<locals>.<lambda>Y  s    1rW   )r   rr  rs  r   r   r   r+   rZ   rI   rD   r   r   r   )r   rw  rd   r   r   r   r   predictionsr   r   check_3ds              rU   "test_cross_val_predict_input_typesr!  )  s   ;D99dkkqAQHOOQ$B$L1M
e!
4C $CA.K&&& $C=AK(((#C15K{((&1 $C=AK{(((3 /J
Z
0C#CQXXZ@K
Z
0C#CAHHJ7K $+.	
	
"	K $+.		
"	K Q2::D$H
X
.C#Cq1K{((&1rW   c                    	 t         t         fg} 	 ddlm}m} | j	                  ||f       | D ]C  \  	 t               	t              }}fd}	fd}t        ||      }t        |||d       E y # t
        $ r Y Tw xY w)Nr   rX  c                     t        |       S r`   r\  r]  s    rU   r   z/test_cross_val_predict_pandas.<locals>.<lambda>m  r_  rW   c                     t        |       S r`   r\  ra  s    rU   r   z/test_cross_val_predict_pandas.<locals>.<lambda>n  rc  rW   rd  r   r   )
rE   re  rY  rZ  r  rf  rd   r   rD   r+   rg  s
           @@rU   test_cross_val_predict_pandasr%  _  s     ]+,E,fi() ).$
$&q):b>e<: <H#tUq1 ).  rn  c                  :   t        dddddd      \  } }d|d<   t        dd      }t        d      }t        |j	                  | |            \  }}t        || ||d	
      }||d      d   dk(  sJ t        j                  ||d      d d df   dk(        sJ t        j                  ||d      d d ddf   dkD        sJ t        j                  ||d      dkD        sJ t        |j                  d      t        j                  |j                        d       y )Nr  r   r   r   )r  r  n_redundantr  n_clusters_per_classr   r  r   r  r  r   r  axis   )decimal)r   r   r*   r   r  r+   r   r	  rH   r  r  rZ   )rd   r   r   r   r+  r,  
yhat_probas          rU   !test_cross_val_predict_unbalancedr/  s  s"   DAq AaD
!K
@C	!	$Brxx1~&KE4"31OLJT!W:a=A66*T!W%ad+q011166*T!W%a1f-122266*T!W%)***jnn!n4bggagg6FPRSrW   c                      t               } t        j                  j                  d      }|j	                  dd      }t        | |d dd      }t        |d d df   |       t        | |d dd      }t        ||       y )	Nr  r  r   r   re   )r   r   r  r   r   )r   r   randomRandomStaterandr+   rF   )mock_classifierrngrd   y_haty_hat_probas        rU   test_cross_val_predict_y_noner8    ss    $&O
))


#CbAoqDQyQEAadGU##dqK A{#rW   c                    t               }|j                  |j                  }}t               }d | t	        j
                  |j                  d               i}t        ||||d      }t        |t	        j                  d             y )Nr   r   r   )r   r   )
r   rr  rs  r   r   r  rZ   r,   rI   r  )r   rw  rd   r   r   r  r   s          rU   &test_cross_val_score_sparse_fit_paramsr:    sf    ;D99dkkqA

C(-qwwqz8J*KLJQ*;Aq"''!*%rW   c                     d} d}t        | dddddd      \  }}t        | |dz
  |z  z        }dD ]   }t        j                  d	      5 }t	        |||t        |
      t        j                  ddd      |d      \  }}}	}
}d d d        t              dkD  rt        d|d   j                  z        j                  dk(  sJ 	j                  dk(  sJ 
j                  dk(  sJ j                  dk(  sJ t        t        j                  ddd             t        |j                  d      t        j                  ddd             t        |	j                  d      t        j                  ddd             |
j                  dk(  sJ |j                  dk(  sJ t        j                  d	      5 }t	        |||t!        ||       t        j                  ddd      |      \  }}}d d d        t        |      dkD  rt        d|d   j                  z        t        |       t        |	        y # 1 sw Y   xY w# 1 sw Y   [xY w)Nr  r   r   r   r   r  r  r  r'  rR  r(  r   FTTr  r  皙?r   r   )r   rR   rC   return_timesUnexpected warning: %r)r   r   rQ  r*  ffffff?r  r  r  )r   rR   rC   )r   rN   r  r  r.   r%   r   linspacer   RuntimeErrormessagerZ   rI   rH   r  rq  r4   )r  r  rd   r   r   shuffle_trainwrR   train_scorestest_scores	fit_timesscore_timestrain_sizes2train_scores2test_scores2s                  rU   test_learning_curverO    sF   IHDAq 'yX\X4M'NOI&$$D1Q (+KKS"5%! 2  q6A:7!A$,,FGG!!W,,,  G+++')))  G+++;Ar2(>?!,"3"3"3";R[[cSU=VW!+"2"2"2":BKKSRT<UV )+++  I--- $$D1Q8F"H	JKKS"5%95L- 2 q6A:7!A$,,FGG!->!,<[ '11@ 21s   
8H526I5H?	I	c                     t        ddddddd      \  } }t        d      }t        || d dt        j                  dd	d
            \  }}}t        |t        j                  ddd
             t        |j                  d      t        j                  dd	d
             t        |j                  d      t        j                  dd	d
             y )Nr  r   r   r   r<  rQ  r   r>  r   r   )r   r   rR   r*  rA  )r   rN   r.   r   rC  rI   rH   r  rd   r  r   rR   rH  rI  s         rU    test_learning_curve_unsupervisedrR    s    DAq 'r*I-;1Cb0I.*K{ {BKK2r$:;l//Q/7S#r9RSk..A.6Cb8QRrW   c            	         t        ddddddd      \  } }t        d      }t        j                  }t	               t        _        	 t        || |dd      \  }}}t        j                  j                         }t        j                  j                          |t        _        d	|v sJ y # t        j                  j                         }t        j                  j                          |t        _        w xY w)
Nr  r   r   r   r<  rQ  r   )r   verbosez[learning_curve])r   rN   sysstdoutr   r.   getvalueclose)rd   r   r   
old_stdoutrR   rH  rI  outs           rU   test_learning_curve_verboser[    s    DAq 'r*IJCJ 1?q!12
.\; jj!!#


$$$	 jj!!#


s   B' 'A	C0c            	          t        ddddddd      \  } }t        d      }t        j                  t              5  t        || |d       d d d        y # 1 sw Y   y xY w)Nr   r   r   r<  T)exploit_incremental_learningr   rN   r   r   r   r.   rd   r   r   s      rU   5test_learning_curve_incremental_learning_not_possibler`  
  sW    DAq 'q)I	z	"y!QTJ 
#	"	"s   AAc                     t        ddddddd      \  } }t        d      }dD ]  }t        || |dd	t        j                  d
dd      |      \  }}}t        |t        j                  ddd             t        |j                  d      t        j                  ddd             t        |j                  d      t        j                  d
dd              y )Nr  r   r   r   r<  rQ  r=  r   Tr>  r   r   )r   r]  rR   rC   r*  rA  r   rx   r.   r   rC  rI   rH   r  )rd   r   r   rF  rR   rH  rI  s          rU   (test_learning_curve_incremental_learningrc    s    DAq 2"5I&1?)-Cb1!2
.\; 	;Ar2(>?!,"3"3"3";R[[cSU=VW!+"2"2"2":BKKSRT<UV 'rW   c                     t        ddddddd      \  } }t        d      }t        || d ddt        j                  d	d
d            \  }}}t        |t        j                  ddd             t        |j                  d      t        j                  dd
d             t        |j                  d      t        j                  d	d
d             y )Nr  r   r   r   r<  rQ  r   Tr>  r   r   )r   r   r]  rR   r*  rA  rb  rQ  s         rU   5test_learning_curve_incremental_learning_unsupervisedre  4  s    DAq 2"5I-;	
%)KKS"-.*K{ {BKK2r$:;l//Q/7S#r9RSk..A.6Cb8QRrW   c            	         t        ddddddd      \  } }t        j                  ddd      }t        dd d	
      }t	        || ||dd      \  }}}t	        || |d|d	      \  }}}	t        ||       t        |j                  d      |j                  d             t        |j                  d      |	j                  d             y )Nr  r   r   r   r<  r  r   r   Fmax_itertolrC   r   T)rR   r   r]  )r   rR   r]  r*  )r   r   rC  r   r.   rI   rH   r  )
rd   r   rR   r   train_sizes_inctrain_scores_inctest_scores_inctrain_sizes_batchtrain_scores_batchtest_scores_batchs
             rU   <test_learning_curve_batch_and_incremental_learning_are_equalrp  L  s    DAq ++c3*K+QD%PI9G		%):6O% @N		%*@<)+< (9:1%'9'>'>A'>'F !$&7&<&<!&<&DrW   c            	         t        ddddddd      \  } }t        d      }t        j                  t              5  t        || |dddg       d d d        t        j                  t              5  t        || |dd	d
g       d d d        t        j                  t              5  t        || |dddg       d d d        t        j                  t              5  t        || |dddg       d d d        t        j                  t              5  t        || |dddg       d d d        y # 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   bxY w# 1 sw Y   y xY w)Nr  r   r   r   r<  rQ  r   r   rR   r  r   r>  g?   r^  r_  s      rU   0test_learning_curve_n_sample_range_out_of_boundsrt  s  s   DAq 'r*I	z	"y!Q11a&A 
#	z	"y!Q13*E 
#	z	"y!Q13*E 
#	z	"y!Q11b'B 
#	z	"y!Q11b'B 
#	" 
#	"	"	"	"	"	"	"	"	"s;   D&.D2"D>E

E&D/2D;>E
EEc                     t        ddddddd      \  } }t        d      }d}t        j                  t        |      5  t        || |dt        j                  dd	d      
      \  }}}d d d        t        ddg       y # 1 sw Y   xY w)Nr   r   r   r   r<  zzRemoved duplicate entries from 'train_sizes'. Number of ticks will be less than the size of 'train_sizes': 2 instead of 3.r   gQ?r   rr  )	r   rN   r   r   r  r.   r   rC  rI   )rd   r   r   r   rR   r  s         rU   1test_learning_curve_remove_duplicate_sample_sizesrv    s    DAq 'q)I	G  
nO	<*q!r{{4a/H
Q 
= {QF+	 
=	<s   *A??Bc                     t        ddddddd      \  } }t        d      }t        d      }t        || ||t	        j
                  d	d
d            \  }}}t        |t	        j
                  ddd             t        |j                  d      t	        j
                  dd
d             t        |j                  d      t	        j
                  d	d
d             y )Nr  r   r   r   r<  rQ  r   r  r>  r   r   rr  r*  rA  )	r   rN   r%   r.   r   rC  rI   rH   r  )rd   r   r   r   rR   rH  rI  s          rU   (test_learning_curve_with_boolean_indicesrx    s    DAq 'r*I		B-;1aBBKKS",E.*K{ {BKK2r$:;l//Q/7S#r9RSk..A.6Cb8QRrW   c                     t        j                  ddgddgddgddgd	d
gddgddgddgddgddgddgd	d
gddgddgddgg      } t        j                  g d      }t        j                  g d      }t        dd d      }t        d      }t	        || ||dt        j
                  ddd      |dd	      \  }}}t        |j                  d      t        j                  g d             t        |j                  d      t        j                  g d              t        j                  t              5  t	        || ||dt        j
                  ddd      |d!"       d d d        t	        || ||dt        j
                  ddd      |ddd#
      \  }}	}
t        |	j                  d      |j                  d             t        |
j                  d      |j                  d             y # 1 sw Y   xY w)$Nr   r   r   r   r   r  r  r     r,                       rQ  	   r   )r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   )r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   Frg  r  333333?r   T)r   r   rR   r  rC   r   r*  )r  r  Iq?)r  r  r  r   )r   r   rR   r  r   )r   r   rR   r  rC   r   r]  )r   r  r   r#   r.   rC  rH   r  r   r   r   )rd   r   r  r   r   rm  rn  ro  rj  rk  rl  s              rU    test_learning_curve_with_shuffler    s&    	FFFFHHHHHFGHHHH	
	A& 	>?AXXCDF ,QD%PI	Q	B?M		KKS!,
@<)+< Q'2I)J A&1I(J 
z	"Ca0		
 
# :H		KKS!,%):6O% 1%'9'>'>A'>'F !$&7&<&<!&<&D7 
#	"s   3)G++G4c                      t        j                  d      j                  dd      } t        j                  dgdz  dgdz  z         }t	        d      }d}t        j                  t        |	      5  t        || |d
       d d d        d}t        j                  t        |	      5  t        || |d
dt        j                  d      i       d d d        t        || |d
dt        j                  d      i       y # 1 sw Y   zxY w# 1 sw Y   ;xY w)Nr  r   r   r   r   Tr  r  r   r   r   z/sample_weight.shape == \(1,\), expected \(2,\)!r   )r   r  )r   r  r   r  rD   r   r   r   r.   r   r  r  s       rU   test_learning_curve_fit_paramsr    s    
		#r2&A
!qA37"#A
D
9C4G	~W	5sAqg6 
6 AG	z	1A7QR7T	
 
2 QwORWWR[3Q 
6	5 
2	1s   /C8$&D8DDc                  R   t        ddddddd      \  } }t        ddg      }d}t        j                  t        |	      5  t        || |d
dt        j                  ddd      d       d d d        d}t        j                  t        |	      5  t        || |d
dt        j                  ddd      ddt        j                  d
      i       d d d        t        || |d
dt        j                  ddd      ddt        j                  d      i       y # 1 sw Y   xY w# 1 sw Y   SxY w)Nr  r   r   r   r<  rQ  r   z9Expected fit parameter\(s\) \['sample_weight'\] not seen.r   r   Tr>  r   r   r   )r   r]  rR   r   z2Fit parameter sample_weight has length 3; expected)r   r]  rR   r   r  )	r   rx   r   r   r   r.   r   rC  r  )rd   r   r   r  s       rU   3test_learning_curve_incremental_learning_fit_paramsr    s   DAq 2"6GHIJG	~W	5)-Cb1	
 
6 CG	~W	5)-Cb1'4		
 
6 		%)KKS"-#RWWQZ0	1 
6	5 
6	5s    (D>DDD&c            	         t        ddddddd      \  } }t        j                  ddd      }t        j                  d      5 }t        t               | |d|d	      \  }}d d d        t              dkD  rt        d
|d   j                  z        t        j                  d      |       t        j                  d      d|z
         y # 1 sw Y   nxY w)Nr   r   r   r<  r   Tr  r   
param_nameparam_ranger   r@  r*  )r   r   rC  r  r  r0   r   r   rD  rE  rH   r  )rd   r   r  rG  rH  rI  s         rU   test_validation_curver  E  s    DAq ++aB'K		 	 	-$4&(#%
!k 
. 1vz3adllBCCl//Q/7Ek..A.6KH 
.	-s   CCc            	          t        ddddddd      \  } }t        j                  ddd      }t        t	               | |d|d      \  }}y )Nr   r   r   r<  r   r   r  )r   r   rC  r0   r   )rd   r   r  r  s       rU   %test_validation_curve_clone_estimatorr  `  s\    DAq ++aB'K-/		DAqrW   c                  \   d} d}t        dd      \  }}t        t        dd      ||dg dt        || 	      
      }t	        t        j                  t        j                  |      dd d f   d        t        t        dd      ||dg dt        |d      
      }t	        t        j                  t        j                  |      dd d f   d        t        t        dd      ||dg dt        |      
      }t	        t        j                  |      t        j                  |             y )Nr  r   r   r  r  r  r   )r>  r>  r  r  rB  r  )r   r   r   r   r   Tr  rC   r  )
r   r0   r<   r4   rH   r   vsplithstackr%   r  )r  r  rd   r   scores1scores2scores3s          rU   +test_validation_curve_cv_splits_consistencyr  v  s   IH1=DAq8!,		(H	BG ryy7);L!O)LaPQ8!,		((D1G ryy7);L!O)LaPQ8!,		((#G bhhw/'1BCrW   c                  8   t        j                  d      j                  dd      } t        j                  dgdz  dgdz  z         }t	        d      }d}t        j                  t        |	      5  t        || |d
g dd       d d d        d}t        j                  t        |	      5  t        || |d
g dddt        j                  d      i       d d d        t        || |d
g dddt        j                  d      i       y # 1 sw Y   xY w# 1 sw Y   ?xY w)Nr  r   r   r   r   Tr  r  r   	foo_param)r   r   r   r   )r  r  r   r  r   )r  r  r   r  )r   r  r   r  rD   r   r   r   r0   r   r  r  s       rU    test_validation_curve_fit_paramsr    s    
		#r2&A
!qA37"#A
D
9C4G	~W	5"!	
 
6 AG	z	1"!'4	
 
2 		#RWWR[1+ 
6	5 
2	1s   /D(*DDDc                  Z   t         j                  j                  d      } t        j                  d      }| j	                  |       t        |d      sJ t        t        j                  |d      d      rJ d|d<   t        |d      rJ t        t        j                  |df      d      rJ y )Nr   r     )r   r1  r2  r  rC   r1   deleter  )r5  r  s     rU   test_check_is_permutationr    s    
))


"C
		#AKKN C((($RYYq"%5s;;;AaD$Q,,, %RYY1v%6<<<<rW   c                     t        ddddd      \  }} | |      } | |      }t        t        d            }t        |||d	      }t        |||d	      }|j	                         }t        ||       y )
Nr   r   FT)rR  n_labelsallow_unlabeledreturn_indicatorr   r  rS  r   r   )r   r6   r<   r+   r8  rH   )r  rd   r   r   y_sparseclassifr   preds_sparses           rU   (test_cross_val_predict_sparse_predictionr    s~     *DAq QHQH!#X"67Ggq!3E$WhRHL'')LlE2rW   c           
         t        dd      }|j                  dk(  r|dk(  rt        |      fnt        |      df}n|j                  }t	        j
                  |      }|j                  ||      D ]>  \  }}t        |       j                  ||   ||         }  t        | |      ||         ||<   @ ||dz   |dz
  |j                  d      fD ]  }	t        t        | ||	||      |        y	)
z@Helper for tests of cross_val_predict with binary classificationr   Fr  r   r  r   strr  N)r%   r   r   rZ   r   rt  r  r   r]   getattrastyperF   r+   )
r#  rd   r   r  r   	exp_shapeexpected_predictionsr+  r,  tgs
             rU   check_cross_val_predict_binaryr    s    	5	)B 	vv{!'+>!>SVISVQK	GG	88I.xx1~tCjnnQuXqx0%9WS&%9!D'%BT" &
 !a%Q0c1bB?AU	
 1rW   c           
         t        dd      }t        j                  t        j                        j                  }||dd}t        j
                  t        |      t        t        |            f||   t        j                        }t        j                  |d      \  }}	|j                  ||	      D ]l  \  }
}t        |       j                  ||
   |	|
         }  t        | |      ||         }t        j                  |	|
         }||t        j                  ||      <   n ||d	z   |d
z
  |j                  d      fD ]  }t        t!        | ||||      |        y)zDHelper for tests of cross_val_predict with multiclass classificationr   Fr  r   r  r  r   rp  Treturn_inverser   r   r  r  N)r%   r   finfor  minr  r   r   r   r  r   r]   r  ix_r  rF   r+   )r#  rd   r   r  r   	float_mindefault_valuesr  r  y_encr+  r,  
fold_preds
i_cols_fitr  s                  rU   "check_cross_val_predict_multiclassr     s4   	5	)B $((I&&N
 77	QSV~f5RZZ yy40HAuxx5)tCjnnQuXuU|4)WS&)!D'2
YYuU|,
9CRVVD*56	 * !a%Q0c1bB?AU	
 1rW   c                    t        dd      }t        j                  t        j                        j                  }||dd}|j
                  d   }g }t        |      D ]y  }	t        t        |dd|	f               }
|
dk(  r|d	k(  rt        |      f}nt        |      |
f}|j                  t        j                  |||   t        j                  
             { t        |j
                  d         D cg c]8  }t        j                  |dd|f   d      d   ddt        j                  f   : }}t        j                  |d      }|j                  ||      D ]  \  }}t        |       j!                  ||   ||         }  t#        | |      ||         }t        |      D ]`  }	t        j                  ||   dd|	f         }||	   j$                  dk(  r||	   ||	   |<   @t        j&                  ||      }||	   ||	   |<   b  ||dz   |dz
  |j)                  d      fD ]V  }t+        | ||||      }t        |      t        |      k(  sJ t        t        |            D ]  }t-        ||   ||           X yc c}w )zCheck the output of cross_val_predict for 2D targets using
    Estimators which provide a predictions as a list with one
    element per class.
    r   Fr  r   r  r   Nr   r  rp  Tr  r*  r  r  )r%   r   r  r  r  rZ   r   r   r   r  r  r   r   concatenater  r   r]   r  r   r  r  r+   rF   )r#  rd   r   r  r   r  r  	n_targetsexpected_predsi_coln_classes_in_labelr  r  
y_enc_colsr  r+  r,  r  	fold_colsidxr  cv_predict_outputs                         rU   "check_cross_val_predict_multilabelr    sm   
 
5	)B $((I&&N
 
INy! Qq%x[!12"v1D'DQ	IQ!34IGGI~f5RZZH	
 " qwwqz""A 			!AqD'$/21bjj=A"   NN:A.Exx5)tCjnnQuXuU|4)WS&)!D'2
9%E		%,q%x"89Ie$))Q..8.?u%d+ffT9--7->u%c* & * !a%Q0-c1bBO$%^)<<<<s,-.A-a0.2CD / 1%s   3=I<c                 N    t        dd      \  }}dD ]  }t        | |||        y )Nr   r   )rR  r   r  r   r  )r   r  )r#  rd   r   r  s       rU   *check_cross_val_predict_with_method_binaryr  P  s-     ;DAqM&sAq&9 NrW   c                     t               }|j                  |j                  }}t        ||d      \  }}dD ]  }t	        | |||        y )Nr   r   r  )r   rr  rs  rC   r  )r#  rw  rd   r   r  s        rU   .check_cross_val_predict_with_method_multiclassr  X  sE    ;D99dkkqA1aa(DAqM*31f= NrW   c                  X    t        t        d             t        t        d             y )Nr  r  )r  r   r  rv   rW   rU   "test_cross_val_predict_with_methodr  `  s     ./A/UV2+.rW   c                      t               } | j                  | j                  }}t        ||d      \  }}dD ]  }t	        dd      }t        ||||        y )Nr   r   r  log_lossr   )lossr   )r   rr  rs  rC   r   r  )rw  rd   r   r  r#  s        rU   &test_cross_val_predict_method_checkingr  g  sS     ;D99dkkqA1aa(DAqM!<*31f= NrW   c                      t               } | j                  | j                  }}t        ||d      \  }}t	        t        dd      dddgid	
      }dD ]  }t        ||||        y )Nr   r   r  r  r)  r   r>  r   r   r   r  )r   rr  rs  rC   r"   r   r  )rw  rd   r   r#  r  s        rU   /test_gridsearchcv_cross_val_predict_with_methodr  r  si    ;D99dkkqA1aa(DAq
;?#QxUVC N*31f= NrW   c                      d} d}t        | d|dd      \  }}t        t        dd	            }d
D ]  }t        ||||        y )Nr  r   r   r   r  r  r  rR  r  r   r  r   )r  r   )r   r  r  )r   r6   r   r  )n_samprR  rd   r   r#  r  s         rU   1test_cross_val_predict_with_method_multilabel_ovrr  }  sU     FI)1	aVXDAq 0RST
UC8&sAq@ 9rW   c                       e Zd Zd Zy)RFWithDecisionFunctionc                     | j                  |      }d}t        |t              sJ |       |D cg c]  }|j                  d   dk(  r	|d d df   n|! }}|S c c}w )Nz?This helper should only be used on multioutput-multiclass tasksr   r   r   )r   r   r   rZ   )rT   rd   probsmsgr  s        rU   r  z(RFWithDecisionFunction.decision_function  sf    ""1%O%&++&=BCUQWWQZ1_1b5!3UC Ds   $AN)rr   rs   rt   r  rv   rW   rU   r  r    s    rW   r  c                      d} t        dd| dd      \  }}|d d dfxx   |d d df   z  cc<   d	D ]P  }t        dd
      }t        j                         5  t        j                  d       t        ||||       d d d        R y # 1 sw Y   ]xY w)Nr   r  r   r   r  r  r   r   r   r  r  n_estimatorsr   ignorer  )r   r  r  r  simplefilterr  )rR  rd   r   r  r#  s        rU   0test_cross_val_predict_with_method_multilabel_rfr    s     I)Y1SUDAq adGqAwGM$!!D$$&!!(+.sAqH '& N&&s   %BB	c                  V   t         j                  j                  d      } | j                  ddd      }t        j                  g d      }t        d      }dD ]B  }t        j                         5  t        j                  d	       t        ||||       d d d        D y # 1 sw Y   OxY w)
Nr   r   )r|  r   r  )r   r   r   r   r   r   r   r   r   r   r   r   r   r   r  r  r  r  )
r   r1  r2  normalr  r   r  r  r  r  )r5  rd   r   r#  r  s        rU   -test_cross_val_predict_with_method_rare_classr    s     ))


"C

1ah
'A
;<A
K
0CM$$&!!(+.sAq&A '& N&&s   0$BB(	c                  t   t         j                  j                  d      } | j                  ddd      }t        j                  ddgddgddgddgddgg      }dD ]P  }t        dd      }t        j                         5  t        j                  d	       t        ||||
       d d d        R y # 1 sw Y   ]xY w)Nr   r   )r   r   r  r   )r   r  r   r  r  r  )
r   r1  r2  r  r  r  r  r  r  r  )r5  rd   r   r  r#  s        rU   ;test_cross_val_predict_with_method_multilabel_rf_rare_classr    s     ))


"C

1ag
&A
1a&1a&1a&1a&1a&9:A8$!!D$$&!!(+.sAqH '& 9&&s   >%B..B7	c                    t        j                  t        |      |g      }t        ||      }|j	                  | |      D ]  \  }}	|j                  | |   ||           || |	         }
|dk(  r!t        j                  t        |	      |f      }nHt        j                  t        |	      |ft        j                  |j                        j                        }|
|d d |j                  f<   |||	<    |S )Nr   )r   rt  r   r  r  r]   r  r  rq  r  classes_)rd   r   r   classesr#  r  r  funcr+  r,  expected_predictions_exp_pred_tests               rU   get_expected_predictionsr    s    88SVW$563Dxx1~t%!E(# $QtW_$HHc$i%9:MGGTG$bhh/C/I/I&J&N&NM *?ao&%2T" &  rW   c            	         t        j                  d      j                  dd      } t        j                  t	        d      D cg c]  }|dz  	 c}      }d}t        d      }t        d      }t               }g d}|D ]  }t        d	
      }	t        |	| |||      }
t        | ||||	|      }t        ||
       t        |	| |||      }
t        | ||||	|      }t        ||
       t        t        j                  t	        d      d      d      }t        |	| |||      }
|j                  |      }t        | ||||	|      }t        ||
        y c c}w )Nr  r  r   r   r   r  r   r  r  r  r  r   r   )r   r  r   r  r   r%   r:   r   r+   r  rH   rC   r  fit_transform)rd   r{   r   r  kfold3kfold4lemethodsr  r#  r  r  s               rU   #test_cross_val_predict_class_subsetr    sW   
		#sA&A
5:.:a!r':./AGAFAF	BIG 4 (Q&VL  8q&'3 
 	""6D (Q&VL7q&'3 
 	""6D BIIeBi,1='Q&VLQ7q&'3 
 	""6D5  /s   Ec                     t               } | j                  | j                  }}t               }t	        j
                  dd      }|j                  d       |j                          t        j                  |j                  t        j                        t        j                  |j                  ddt        j                        	 t        |||fd	
       t        j                  t              5  t        |||fd
       d d d        d\  t!        d      D ]#  }	 t#        j$                  |j                          y  y # 1 sw Y   @xY w# t&        $ r t)        d       Y Iw xY w# d\  t!        d      D ]=  }	 t#        j$                  |j                          w # t&        $ r t)        d       Y ;w xY w w xY w)NwbF)moder  s   Hello world!!!!!rp  rv   r)rZ   r  rq  c                     S r`   rv   )r#  rd   r   rl   s      rU   r   z#test_score_memmap.<locals>.<lambda>  s    UrW   r   c                     S r`   rv   )r#  rd   r   r   s      rU   r   z#test_score_memmap.<locals>.<lambda>  s    rW   rp   r   r   )r   rr  rs  r   tempfileNamedTemporaryFilewriterX  r   memmapnamer  r,   r   r   r   r   osunlinkOSErrorr   )rw  rd   r   r   tfr  rl   r   s         @@rU   test_score_memmapr    sL   ;D99dkkqA

C		$	$$u	=BHH !HHJYYrwwbjj1FIIbggRcDEQ+BC]]:&CA/GH '
 #qA		"''"  '&  c
 #qA		"''" c
	 sZ   +E+ 3EE+ "EE
E+ E('E(+F= F!F=!F8	5F=7F8	8F=c                  8   t         t         fg} 	 ddlm}m} | j	                  ||f       | D ][  \  t               }|j                  |j                  }} |       |      }}fd}fd}	t        ||	      }
t        |
||       ] y # t
        $ r Y lw xY w)Nr   rX  c                     t        |       S r`   r\  r]  s    rU   r   z4test_permutation_test_score_pandas.<locals>.<lambda>,  r_  rW   c                     t        |       S r`   r\  ra  s    rU   r   z4test_permutation_test_score_pandas.<locals>.<lambda>-  rc  rW   rd  )rE   re  rY  rZ  r  rf  r   rr  rs  rD   r/   )rh  rY  rZ  rw  rd   r   ri  rj  rk  rl  r   r^  rb  s              @@rU   "test_permutation_test_score_pandasr    s     ]+,E,fi() ).$
${yy$++1&q):a=e<: <HsD%0 ).  s   B 	BBc                  F   t        t         j                        } t        j                  dd      }t	        | |d t	               d d dd d d 
      }d|d<   t        j                  t        d      5  t        d
i | d d d        | j                         d	k(  sJ y # 1 sw Y   xY w)Nr   r   r   )
r   rd   r   r  r+  r,  rT  
parametersr  score_paramsr   r   z%Failing classifier failed as requiredr   r  rv   )
r5   FAILING_PARAMETERr   r  r   r   r   r   r2   rl   )failing_clfrd   fit_and_score_argss      rU   test_fit_and_score_failingr  2  s    #$5$G$GHK
		!RA

v )0}%	z)P	Q,+, 
R #%%% 
R	Qs   -BB c                      t        dd      \  } }t        dd      }t        t               j	                  |             \  }}t        || |t               ||ddddd d d	
      }t        di |}|d   |d   k(  sJ y )Nr  r   r  r  r  r  r>  )rh  ri  T)r   rd   r   r  r+  r,  rT  r  r  r  return_parametersr  rv   )r   r<   nextr)   r  r   r2   )rd   r   r   r+  r,  r  results          rU   test_fit_and_score_workingr  L  s    !<DAq
XA
.C|~++A./KE4

v #C0 101F,#5l#CCCCrW   c                   $    e Zd ZddZddZddZy)DataDependentFailingClassifierNc                     || _         y r`   max_x_value)rT   r  s     rU   rV   z'DataDependentFailingClassifier.__init__c  s
    &rW   c                 `    || j                   kD  j                         }|rt        d| d      y )NzClassifier fit failed with z values too high)r  r  r   )rT   rd   r   num_values_too_highs       rU   r]   z"DataDependentFailingClassifier.fitf  s@     4#3#3388:-.A-BBRS  rW   c                      y)Nr  rv   rj   s      rU   rl   z$DataDependentFailingClassifier.scorem  s    rW   r`   rp   )rr   rs   rt   rV   r]   rl   rv   rW   rU   r  r  b  s    'rW   r  r   c                 \   t        d      }t        j                  dd      }t        j                  d      }|||g}d| d}d}t	        j
                  d	|d
    d| t        j                        }t        j                  t        |      5  t        |i | d d d        y # 1 sw Y   y xY w)Nr  r  r   r   r  r   r   r   z8ValueError: Classifier fit failed with 1 values too highzh2 fits failed.+total of 3.+The score on these train-test partitions for these parameters will be set to r   z.+flagsr   )r  r   r  r  recompileDOTALLr   r   r   r-   )r   r  rd   r   cross_validate_argscross_validate_kwargsindividual_fit_error_messager   s           rU   -test_cross_validate_some_failing_fits_warningr+  q  s     1Q?K
		!RA

A&1-#$[A 	C ! jj%m45R8T7UW iiO 
&o	>+E/DE 
?	>	>s   B""B+c                 j   t        t         j                        }t        j                  dd      }t        j                  d      }|||g}d| d}d}t        j                  d| t
        j                        }t        j                  t        |	      5  t        |i | d d d        y # 1 sw Y   y xY w)
Nr   r   r  r  r"  z1ValueError: Failing classifier failed as requiredz4All the 7 fits failed.+your model is misconfigured.+r#  r   )r5   r  r   r  r  r%  r&  r'  r   r   r   r-   )r   r  rd   r   r(  r)  r*  rU  s           rU   *test_cross_validate_all_failing_fits_errorr-    s     $$5$G$GHK
		!RA

A&1-#$[A#V JJB+,. iiM 
z	7+E/DE 
8	7	7s   B))B2c                     t        |      r`   )r   )r   rd   r   	error_msgs       rU   _failing_scorerr0    s    
Y
rW   zignore:lbfgs failed to converger   c           	         t        d      \  }}t        d      j                  ||      }d}t        t        |      }| dk(  r6t        j                  t        |      5  t        |||d	|| 
       d d d        y d|  }t        j                  t        |      5  t        |||d	|| 
      }t        ||        d d d        y # 1 sw Y   y xY w# 1 sw Y   y xY w)NTr   r   rh  "This scorer is supposed to fail!!!r/  r   r   r   )r   r   r   r   )r   r   r]   r   r0  r   r   r   r,   r   r   rF   )r   rd   r   r   r/  failing_scorerwarning_msgr   s           rU   #test_cross_val_score_failing_scorerr7    s    
 %DAq
a
(
,
,Q
2C4I_	BNg]]:Y7Qa[ 87//:m= 	 \\+[9$Qa[F FK0	 :9 87 :9s   CCC
Cr0  with_multimetricc           
      L   t        d      \  }}t        d      j                  ||      }d}t        t        |      }|rt        t              }|||d}	n|}	| dk(  r7t        j                  t        |	      5  t        |||d
|	||        d d d        y d|  }
t        j                  t        |
	      5  t        |||d
|	||       }|D ]7  }d|v sd|v r||   D ]  }t        |t              rJ  )t        ||   |        9 	 d d d        y # 1 sw Y   y xY w# 1 sw Y   y xY w)NTr   r   r2  r3  r4  )score_1score_2score_3r   r   r   )r   r   r0  r   r   r3   _score_2)r   r   r]   r   r0  r   r   r   r   r   r-   r   r   r   ri   rF   )r   r0  r8  rd   r   r   r/  r5  non_failing_scorerr   r6  r  r   r  s                 rU   "test_cross_validate_failing_scorerr?    sC    %DAq
a
(
,
,Q
2C4I_	BN();<%)%
 !g]]:Y7#5' 87//:m= 	 \\+[9$#5'G s?!S( ")A#-a#77#7 ".
 (kB  :9 87 :9s$   9D5DD/DDD#c                      y)Ng;pΈ^@rv   )r  jr=  s      rU   three_params_scorerrB  	  s    rW   z:train_score, scorer, verbose, split_prg, cdt_prg, expectedr   )r   r   r   r   zS\[CV\] END .................................................... total time=   0.\ds)sc1sc2)scorersr   ze\[CV 2/3\] END  sc1: \(train=3.421, test=3.421\) sc2: \(train=3.421, test=3.421\) total time=   0.\dsr   zW\[CV 2/3; 1/1\] END ....... sc1: \(test=3.421\) sc2: \(test=3.421\) total time=   0.\dsc                    t        dd      \  }}t        dd      }	t        t               j	                  |            \  }
}t        |	||||
||d d d |||      }t        d
i | | j                         \  }}|j	                  d      }t        |      dkD  rt        j                  ||d	         sJ y t        j                  ||d         sJ y )Nr  r   r  r  r  )r   rd   r   r  r+  r,  rT  r  r  r  r0  split_progresscandidate_progress
r   r   rv   )r   r<   r  r)   r  r   r2   
readouterrr   r%  r   )capsysr1  r  rT  	split_prgcdt_prgexpectedrd   r   r   r+  r,  r  rZ  r  outliness                   rU   test_fit_and_score_verbosityrQ  
	  s    N !<DAq
XA
.C|~++A./KE4 

& " ('( FCyyH
8}qxx(1+...xx(1+...rW   c            	          d} d }t        j                  t        |       5  t        d d d |d t        j
                         d d d        y # 1 sw Y   y xY w)Nz&scoring must return a number, got Nonec                      y r`   rv   )r   X_tests     rU   two_params_scorerz%test_score.<locals>.two_params_scorerQ	  s    rW   r   )r   rT  r  r  r  r   )r   r   r   r3   r   r   )rU  rU  s     rU   r2  r2  N	  sF    <M 
z	7$	
 
8	7	7s    A

Ac                      d } t        ddd      \  }}t        d      }|j                  ||       t        |||d|       }g d	}|D ]  }d
j	                  |      |v rJ  y )Nc                 b    | j                  |      }t        ||      }|d   |d   |d   |d   dS )N)r   r   rC  )r   r   )r   r   tnfpfntp)re   r   )r   rd   r   rC  cms        rU   rD  zPtest_callable_multimetric_confusion_matrix_cross_validate.<locals>.custom_scorer`	  s:    Qa(hbhbhbhOOrW   (   r   r  )r  r  r   r   r   )r   r   rX  ztest_{})r   r=   r]   r-   r   )rD  rd   r   r#  rO  score_namesr  s          rU   9test_callable_multimetric_confusion_matrix_cross_validater`  _	  sm    P
 KDAq

$CGGAqMQaGJ*K%333 rW   c                  R    t        d      \  } }t        t               | |dd       y)z^Check that regressors with partial_fit is supported.

    Non-regression test for #22981.
    r  r   Tr   )r]  r   N)r   r.   r8   )rd   r   s     rU   *test_learning_curve_partial_fit_regressorsrb  o	  s&    
 +DAq <>1adqQrW   c                    t        dddd|       \  }}t        j                  |      }||   ||   }}t               }d}t	        j
                  t        |      5  t        |||dt        j                  	      ^}}}}d
d
d
       t        d      D ]R  }	t        j                  |	         j                         sJ t        j                  |	         j                         rRJ  t        dj                  d         D ]R  }	t        j                  ||	         j                         rJ t        j                  |	         j                         sRJ  y
# 1 sw Y   xY w)zKChecks for fit failures in `learning_curve` and raises the required warningr  r   r  F)r  rR  r  rC   r   z#10 fits failed out of a total of 25r   r   r"  Nr   r   )r   r   r  r<   r   r   r   r.   r   r   isnanr	  rZ   any)
global_random_seedrd   r   
sorted_idxsvcr   r  r1  r2  r  s
             rU   -test_learning_curve_some_failing_fits_warningri  z	  s8    'DAq AJZ=!J-qA
%C;O	&o	>)7A!*
&;
Q 
? QxxxC()--///xx
3(,,...  Q))!,-88K,-1133388JsO,00222 . 
?	>s   #EEc                 x   t        d      \  }}t        |      }t               }t        dd|       }t	        ||||dd      }d|vsJ t	        ||||dd      }d|v sJ |d   d	   }|d   d
   }t        |      |j                  k(  sJ t        |      |j                  k(  sJ t        |D cg c]  }|j                   c}d       t        |D cg c]  }|j                   c}d       t        |j                  ||            D ]&  \  }	\  }
}t        ||	   |
       t        ||	   |       ( yc c}w c c}w )z<Check the behaviour of `return_indices` in `cross_validate`.Tr   r   )r  rC   r   r   F)r   r   return_indicesindicesr+  r,  r  r  N)r   r;   r   r%   r-   r   r  rI   r  r5  r  )rf  rd   r   r   r   rO  train_indicestest_indicesrl  	split_idxexpected_train_idxexpected_test_idxs               rU   "test_cross_validate_return_indicesrr  	  sM   %DAqaA"$I	46H	IB	1aBqQVWJJ&&&	1aBqQUVJ
"""y)'2Mi(0L},,,|+++MBMMBCHLALLA2F>GQRTU>W:	:&(9=35GH<	24EF ?X CAs   .D2D7c                  D   t        j                  t        d      5  t        t	               t
        t        di        ddd       t        j                  t        d      5  t        t	               t
        t        i i        ddd       y# 1 sw Y   NxY w# 1 sw Y   yxY w)z2Check that we warn about deprecating `fit_params`.z`fit_params` is deprecatedr   r   )r   rd   r   r   r  Nz1`params` and `fit_params` cannot both be provided)r   rd   r   r  r   )	r   r   FutureWarningr-   r>   rd   r   r   r   rv   rW   rU   )test_cross_validate_fit_param_deprecationru  	  sw    	m+G	H!4!6!qQSUV 
I 
M
 	)+qA"R	

 
 
I	H
 
s   !B
 !B
BBenable_slep006	cv_methodc                     t        j                  t        d      5   | t               t        t
        g        ddd       y# 1 sw Y   yxY w)zCheck that we raise an error if `groups` are passed to the cv method instead
    of `params` when metadata routing is enabled.
    z`groups` can only be passed ifr   )r   rd   r   r  N)r   r   r   r>   rd   r   )rw  s    rU   #test_groups_with_routing_validationry  	  s7     
z)I	J)+		
 
K	J	Js   AAc           	          t        j                  d      }t        j                  t        |      5   | t               t        t        t        g              ddd       y# 1 sw Y   yxY w)zMCheck that we raise an error when passing metadata that is not
    requested.z8but are not explicitly set as requested or not requestedr   )metadata)r   rd   r   r   N)	r%  escaper   r   r   r>   rd   r   r   )rw  r  s     rU    test_passed_unrequested_metadatar}  	  sI     iiRSG	z	1)+$		
 
2	1	1s   'A!!A*c                    t               }t        |      j                  dd      }t               }t        |      j	                  dd      }t               }t        |      j                  dd	      }t        t              }t        j                  j                  d
      }|j                  |      }	|j                  |      }
|j                  d
d|      }|j                  |      }|j                  |      }|j                  |      }t        t        t        |d            t         t        |      t"        t               i}t        ||||      }| t"        ur|j%                  |	|
        | |ft        t&        |d||    d|i | t"        urt)        |      sJ |D ]  }t+        |dd|	|
        t)        |      sJ |D ]  }t+        |d||        t)        |      sJ |D ]  }t+        |dd||        y)z]Check that the respective cv method is properly dispatching the metadata
    to the consumer.)registryscore_weightsscore_metadata)r   r{  split_groupssplit_metadata)r  r{  fit_sample_weightfit_metadatar   r   r  )	my_scorerr  r   )r  r  r  r  )r  r  )rd   r   r   r   rl   )objr  split_paramsr   r{  r  )r  r  r  r{  r]   N)rA   r?   set_score_requestr@   set_split_requestr>   set_fit_requestrL   rd   r   r1  r2  r3  randintr-   r   r,   r+   updater   r   rB   )rw  scorer_registryr  splitter_registrysplitterestimator_registryr   r  r5  r  r  r  r  r  r  extra_paramsr   _scorer	_splitter
_estimators                       rU   test_cross_validate_routingr  	  s/     kOo6HH%0@ I F " *;<NN(8 O H ##-?@PP)N Q I QI
))


"CHHY'MXXi(N;;q!Y/LXXi(N+88I&L 	TFZ%PQf-46L !%+!	F ))') 	 	

 

	
 y
!  ))?###"6'#	
 #  !!!&	#		
 ' !"""(
6+!	
 )rW   )ru   r  r%  rU  r  r  	functoolsr   ior   timer   numpyr   r   scipy.sparser   sklearn.baser   r   sklearn.clusterr	   sklearn.datasetsr
   r   r   r   r   r   sklearn.ensembler   sklearn.exceptionsr   sklearn.imputer   sklearn.linear_modelr   r   r   r   r   sklearn.metricsr   r   r   r   r   r   r   r   r    sklearn.metrics._scorerr!   sklearn.model_selectionr"   r#   r$   r%   r&   r'   r(   r)   r*   r+   r,   r-   r.   r/   r0   #sklearn.model_selection._validationr1   r2   r3   $sklearn.model_selection.tests.commonr4   )sklearn.model_selection.tests.test_searchr5   sklearn.multiclassr6   sklearn.neighborsr7   sklearn.neural_networkr8   sklearn.pipeliner9   sklearn.preprocessingr:   r;   sklearn.svmr<   r=   %sklearn.tests.metadata_routing_commonr>   r?   r@   rA   rB   sklearn.utilsrC   sklearn.utils._mockingrD   rE   sklearn.utils._testingrF   rG   rH   rI   sklearn.utils.fixesrJ   rK   sklearn.utils.validationrL   rN   rx   r   r   r   r  rd   r  r   r   r  r   markparametrizer   r   r  r  ru  r-  r  r  rV  filterwarningsrm  r~  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r!  r%  r/  r8  r:  rO  rR  r[  r`  rc  re  rp  rt  rv  rx  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r   r+  r-  r0  r7  r?  rB  rQ  r2  r`  rb  ri  rr  ru  usefixturesry  r}  r  rv   rW   rU   <module>r     s0     	 	 
        ! - "  4 / ( 
 
 
 7    " 
 A G 2 2 / % 5 &  " D  ? 1"] "6(> <" ",	",F 	"R8 R8n BGGGBHH+, RXX,-
BFF1I .9%< :%<P)1LhL& t}5.94AD 4A : 64An?MDF9R>, FG0 H0&5&86 .9 2 : 2F& J&L0 .9? :?D$QH< .95
 :5
p1Wh# #  .922 :22j FG2 H2$T,
$ .9& :&:=zS&%4K W4S0$NC.,,S(K\&-`I6,+D\#L= .93 :3$
*
81Eh:>>>A	3 	I"BI" (%EP4 FG1 H1(&4D,]  4F 5F6 4F 5F.  =>G(<=1 > ?14 =>G(<=-e}=+eT];:C < > > ?:Cz @ $	
  3<OP ?
	
  3<OP $
	
+ #H/I#H/@
"4 R3BG<

 )*./3DE

 +

 )*./3DE

 +

 )*./3DES
 +S
rW   