
    Kiu                       S SK r S SKrS SKJr  S SKJrJrJr  S SKr	S SK
r
S SKJrJr  S SKJr  S SKJr  S SKJrJr  S SKJr  S S	KJr  S S
KJr  S SKJrJrJrJ r J!r!J"r"J#r#J$r$J%r%J&r&J'r'J(r(J)r)J*r*J+r+J,r,J-r-J.r.J/r/J0r0J1r1  S SK2J3r3J4r4J5r5  S SK6J7r7  S SK8J9r9J:r:  S SK;J<r<  S SK=J>r?  S SK=J@r@JArA  S SKBJCrC  S SKDJErEJFrFJGrGJHrHJIrIJJrJ  S SKKJLrL  S SKMJNrNJOrO  S SKPJQrQ  GSS jrRS rS\
R                  R                  SSS S\	R                  /5      S 5       rW\
R                  R                  SS /S4S S/S4/ S QS4/5      S! 5       rXS" rYS# rZ\
R                  R                  S$5      S% 5       r\\
R                  R                  S$5      S& 5       r]\
R                  R                  S$5      S' 5       r^S( r_\
R                  R                  S)/ S*Q\	R                  " / S+Q/ S,Q/ S-Q/ S.Q/5      4/ S/Q/ S0Q4/5      S1 5       ra\
R                  R                  S)/ S2Q\	R                  " / S.Q/ S.Q/ S3Q/ S3Q/ S4Q/5      4/ S5Q/ S6Q4/5      S7 5       rbS8 rcS9 rdS: reS; rf\
R                  R                  S<\N5      \
R                  R                  S=\O5      S> 5       5       rgS? rh\
R                  R                  S@/ SAQ5      SB 5       riSC rjSD rk\
R                  R                  SE\	R                  " / SFQ5      \	R                  " / SFQ5      SG.SH4\	R                  " / SFQ5      \	R                  " / SIQ5      SG.SJ4\	R                  " / SFQ5      \	R                  " / SKQ5      SG.SL4\	R                  " / SIQ5      \	R                  " / SFQ5      SG.SM4/5      SN 5       rl\
R                  R                  SO\	R                  " / SPQ5      \	R                  " / SQQ5      SG.SR4/5      SS 5       rmST rn\
R                  R                  SUSS/5      SV 5       roSW rp\
R                  R                  SXSYSZ0S[SZ0S\SZS].S^S_S].SZSZS].S^S`S]./5      Sa 5       rq\
R                  R                  SbS^S^S].S^4\	R                  SZS].\	R                  4S`SZS].S`4\	R                  \	R                  S].\	R                  4\	R                  \	R                  4/5      Sc 5       rs\
R                  R                  SbS^S^S].S^4\	R                  SZS].SZ4\	R                  SdS].Sd4\	R                  \	R                  S].\	R                  4\	R                  \	R                  4/5      Se 5       rtSf ruSg rv\
R                  R                  SS S\	R                  /5      \
R                  R                  ShS /S /4/5      \
R                  R                  Si\%\" \&SSj9\/\0/5      Sk 5       5       5       rw\
R                  R                  ShS /S /4/5      \
R                  R                  Si\%\" \&SSj9\/\0/5      Sl 5       5       rxSm rySn rzSo r{Sp r|\
R                  R                  SqSrSs/5      St 5       r}Su r~\
R                  R                  Sv/ SwQ5      Sx 5       rSy rSz rS{ r\
R                  R                  S|/ S}4S~S/S4/SS/S9S 5       rS r\
R                  R                  S/ SQ5      S 5       rS rS rS rS rS rS rS rS rS r\
R                  R                  S$5      S 5       rS rS rS rS rS rS rS r\
R                  R                  SSS/5      S 5       r\
R                  R                  S$5      S 5       r\
R                  R                  S$5      S 5       r\
R                  R                  S$5      \
R                  R                  SSSS\	R                  \	R                  4/5      S 5       5       r\
R                  R                  SS/5      \
R                  R                  Sv/ SQ5      \
R                  R                  SS S\	R                  /5      S 5       5       5       r\
R                  R                  Sv/ SQ5      S 5       r\
R                  R                  SS S\	R                  /5      S 5       rS rS r\
R                  R                  SS S\	R                  /5      S 5       r\
R                  R                  SSS S\	R                  /5      S 5       r\
R                  R                  SSS S\	R                  /5      S 5       r\
R                  R                  SSS S\	R                  /5      S 5       rS rS rS rS r\
R                  R                  S\GRP                  " S/S /S/S //5      S4\GRP                  " S /S/S/S//5      S4\GRP                  " / SQ/ SQ/ SQ/5      S4/5      S 5       rS rS rS rS rS rS rS rS r\
R                  R                  S\	GRd                  \	GRf                  \	GRh                  /5      S 5       r\
R                  R                  S\	GRd                  \	GRf                  \	GRh                  /5      S 5       r\
R                  R                  Sh/ SQ/ SQ4/ SQSS /S S/SS //4/ S Q/ SQ/ SQ/ SQ/4/5      S 5       rS rS rS rS rS rS rS r\
R                  R                  S/ SQ/ SQ4/ SQ/ SQ4/ SQ/ SQ4/5      S 5       r\
R                  R                  Si\)\%\" \&SdSj9\.\/\0\ /5      \
R                  R                  S/ SQ5      S 5       5       r\
R                  R                  S\	R                  " S S/5      \	R                  " SS /5      SZ4\	R                  " S S/5      \	R                  " S S/5      S^4\	R                  " S S/5      \	R                  " S S /5      SZ4\	R                  " S S /5      \	R                  " S S /5      S^4/5      S 5       r\
R                  GR                  \
R                  R                  S\+" \%\	R                  S9\+" \&S\	R                  S9\+" \/\	R                  S9\+" \0\	R                  S9/5      S 5       5       rS rS rS rS rS rS r\
R                  R                  S/ SQ/ SQSS4/ SQ/ SQSS4/ SQ/ SQSS4/ SQ/ SQSS4/ SQ/ SQSS4/ SQSdSd/SdSd/SdSd//SS4/ SQ/ SQ// SQ/ SQ/SS4/ SQ/ SQ/ SQ/ SQ/S S/S4/ SQ/ SQ/ SQ/ SQ/S /S4/	5      S 5       rS r\
R                  R                  S\A" 5       5      GS  5       r\
R                  R                  GS\ \*\4\5/5      \
R                  R                  GSSS/5      \
R                  R                  GSSS/5      \
R                  R                  GS\A" 5       5      GS 5       5       5       5       r\
R                  R                  GS\ \*\4\5/5      \
R                  R                  GSSS/5      \
R                  R                  GS\A" 5       5      GS 5       5       5       rg(      N)partial)chainpermutationsproduct)linalgsparse)hamming)	bernoulli)datasetssvm)config_context)make_multilabel_classification)UndefinedMetricWarning)accuracy_scoreaverage_precision_scorebalanced_accuracy_scorebrier_score_lossclass_likelihood_ratiosclassification_reportcohen_kappa_scoreconfusion_matrixf1_scorefbeta_scorehamming_loss
hinge_lossjaccard_scorelog_lossmake_scorermatthews_corrcoefmultilabel_confusion_matrixprecision_recall_fscore_supportprecision_scorerecall_scorezero_one_loss)_check_targetsd2_brier_scored2_log_loss_score)cross_val_score)LabelBinarizerlabel_binarize)DecisionTreeClassifierdevice)get_namespace)yield_namespace_device_dtype_combinations)MockDataFrame)_array_api_for_testsassert_allcloseassert_almost_equalassert_array_almost_equalassert_array_equalignore_warnings)_nanaverage)CSC_CONTAINERSCSR_CONTAINERS)check_random_stateFc                    U c  [         R                  " 5       n U R                  nU R                  nU(       a  X#S:     X3S:     p2UR                  u  pE[
        R                  " U5      n[        S5      nUR                  U5        X&   X6   p2[        US-  5      n[
        R                  R                  S5      n[
        R                  X'R                  USU-  5      4   n[        R                  " SSSS9n	U	R!                  USU USU 5      R#                  X(S 5      n
U(       a	  U
SS2S	4   n
U	R%                  X(S 5      nX8S nXU
4$ )
zMake some classification predictions on a toy dataset using an SVC

If binary is True restrict to a binary classification problem instead of a
multiclass classification problem
N   %   r      linearT)kernelprobabilityrandom_state   )r   	load_irisdatatargetshapenparanger:   shuffleintrandomRandomStatec_randnr   SVCfitpredict_probapredict)datasetbinaryXy	n_samples
n_featuresprnghalfclfy_pred_probay_predy_trues                k/var/www/html/dynamic-report/venv/lib/python3.13/site-packages/sklearn/metrics/tests/test_classification.pymake_predictionrb   G   s6    $$&AAQxq51GGI
		)A
R
 CKKN4qy1}D ))


"C
a9cJ&6778A ''t!
DC771Ud8QuX.<<QuXFL $AqD)[[5"FuXF<''    c            
      6   [         R                  " 5       n [        U SS9u  pnSSSSS.SS	S
SS.SSSSS.SSSSS.SSSSSS.S.n[        UU[        R
                  " [        U R                  5      5      U R                  SS9nUR                  5       UR                  5       :X  d   eU Hy  nUS:X  a'  [        XV   [        5      (       d   eXV   XF   :X  d   eM0  XV   R                  5       XF   R                  5       :X  d   eXF    H  n[        XF   U   XV   U   5        M     M{     [        US   S   [        5      (       d   e[        US   S   [        5      (       d   e[        US   S   [        5      (       d   e[        US   S   [        5      (       d   eg ) NFrT   rU   g7Mo?gUUUUUU?ghQ?   )	precisionrecallf1-scoresupportUUUUUU?gc1Ƹ?g433333?   g)k??$I$I?   gCFQ?gc?gƢ?K   )ri   rg   rh   rj   g?gD~WG?g]3p?)setosa
versicolor	virginica	macro avgaccuracyzweighted avgT)labelstarget_namesoutput_dictru   rq   rg   rt   rj   )r   rD   rb   r   rH   rI   lenrw   keys
isinstancefloatr3   rK   )irisr`   r_   _expected_reportreportkeymetrics           ra   ,test_classification_report_dictionary_outputr   w   s   D'UCFA
 -)*	
 -*+	
 -)+	
 ++'	
 '++(	
5 OD #yyT../0&&F ;;=O002222*fk51111;/"6666;##%)=)B)B)DDDD).#O$8$@&+fBUV /  oh/<eDDDDok2;?GGGGoh/	:C@@@@ok29=sCCCCrc   zero_divisionwarnrC   c                 <   / SQ/ SQp![         R                  " SS9 n[         R                  " SSS9  [        XU SS9  U S	:X  a8  [	        U5      S
:  d   eU H   nSnU[        UR                  5      ;   a  M    e   O	U(       a   eS S S 5        g ! , (       d  f       g = f)Nabc)r   r   dTrecordalwaysz.+Use `zero_division`)message)r   rx   r   rC   z7Use `zero_division` parameter to control this behavior.)warningscatch_warningsfilterwarningsr   ry   strr   )r   r`   r_   r   itemmsgs         ra   0test_classification_report_zero_division_warningr      s    $oF		 	 	- 	2IJ-T	
 F"v;?"?Oc$,,////  : 
.	-	-s   AB6B
Bzlabels, show_micro_avgTr   rC   r<   c                 x    SS/SS/p2[        X#U SS9nU(       a  SU;   d   eSU;  d   egSU;   d   eSU;  d   eg)a  Check the behaviour of passing `labels` as a superset or subset of the labels.
WHen a superset, we expect to show the "accuracy" in the report while it should be
the micro-averaging if this is a subset.

Non-regression test for:
https://github.com/scikit-learn/scikit-learn/issues/27927
r   rC   T)rv   rx   z	micro avgru   Nr   )rv   show_micro_avgr`   r_   r   s        ra   1test_classification_report_labels_subset_supersetr      s\     VaVF"6&dSFf$$$'''V###&(((rc   c                  2   [         R                  " / SQ/ SQ/5      n [         R                  " / SQ/ SQ/5      n[        X5      S:X  d   e[        X 5      S:X  d   e[        X5      S:X  d   e[        U[         R                  " U5      5      S:X  d   e[        U [         R                  " U 5      5      S:X  d   e[        U [         R                  " U R
                  5      5      S:X  d   e[        U[         R                  " U R
                  5      5      S:X  d   eg )Nr   rC   rC   rC   r   rC   r   r   rC         ?rC   r   )rH   arrayr   logical_notzerosrG   y1y2s     ra   .test_multilabel_accuracy_score_subset_accuracyr      s    	9i(	)B	9i(	)B"!S((("!Q&&&"!Q&&&"bnnR01Q666"bnnR01Q666"bhhrxx01Q666"bhhrxx01Q666rc   c            	      8   [        SS9u  pn[        XS S9u  p4pV[        USS/S5        [        USS/S5        [        US	S
/S5        [        USS/5        0 SS04 H  n[        R
                  " 5          [        R                  " S5        [        X40 UD6n[        USS5        [        X40 UD6n	[        U	SS5        [        X40 UD6n
[        U
S
S5        [        [        X4SS0UD6SU-  U	-  SU-  U	-   -  S5        S S S 5        M     g ! , (       d  f       M  = f)NTrU   averageg\(\?g333333?r<   g)\(?g(\?皙?gRQ?   r   rU   errorbeta      )rb   r!   r4   r5   r   r   simplefilterr"   r#   r   r3   r   )r`   r_   r~   rZ   rfskwargspsrsfss              ra   %test_precision_recall_f1_score_binaryr      s'   't4FA 1NJA!a$q1a$q1a$q1q2r(#
 	8,-$$&!!'* :6:B%b$2f77B%b$2&3F3B%b$2F==f=R"$r	B7 '& .&&s   3B
D


D	z1ignore::sklearn.exceptions.UndefinedMetricWarningc            	         S[        SS/SS/5      :X  d   eS[        SS/SS/5      :X  d   eS[        SS/SS/5      :X  d   eS[        SS/SS/SS9:X  d   eS[        SS/SS/5      :X  d   eS[        SS/SS/5      :X  d   eS[        SS/SS/5      :X  d   eS[        SS/SS/[	        S5      S9:X  d   e[        SS/SS/[	        S5      S9[
        R                  " [        SS/SS/SS95      :X  d   eg )	N      ?rC   r   r           infg     j@)r"   r#   r   r   r|   pytestapprox rc   ra   +test_precision_recall_f_binary_single_classr     s;   
 /1a&1a&1111,1v1v....(Aq6Aq6****+q!fq!f15555/2r(RH5555,Bx"b2222(B8b"X....+r2hRuU|DDDDBx"be=RHr2hS1B   rc   c                  ,   / SQn / SQn[        U [        R                  " S5      S9n[        U[        R                  " S5      S9nX4X#4/n[        U5       H  u  nu  p[	        X/ SQS S9n[        / SQU5        [	        X/ SQSS9n[        [        R                  " / SQ5      U5        S	 H1  nUS
:X  a  US:X  a  M  [        [	        X/ SQUS9[	        XS US95        M3     M     S H  n[        R                  " [        5         [	        X#[        R                  " S5      US9  S S S 5        [        R                  " [        5         [	        X#[        R                  " SS5      US9  S S S 5        M     [        R                  " / SQ/ SQ/5      n [        R                  " / SQ/ SQ/5      n[        XS
SS/S9u  pp[        [        R                  " XU
/5      [        R                  " / SQ5      5        g ! , (       d  f       N= f! , (       d  f       GM.  = f)N)rC      r   r<   )rC   rC   r   r<   r   classes)r   rC   r<   r   r   rv   r   )r   r   r   r   r   macro)microweightedsamplesr   r   )Nr   r   r      r   r   r   rC   r   r   rC   rC   rC   r   rC   r   rv   )      ?rC   竪?)r*   rH   rI   	enumerater#   r4   meanr3   r   raises
ValueErrorr   r!   )r`   r_   
y_true_bin
y_pred_binrE   iactualr   rZ   r   r   r~   s               ra   $test_precision_recall_f_extra_labelsr   &  s    FF		!=J		!=Jz67D(Ff_dS!";VD f_gV!"''*C"DfM 8G)#QVOWUVD'J 8  /( 7]]:&		!gV ']]:&ryyQ/? '& 7 XXy),-FXXy),-F0	1a&JA! !+RXX6G-HI '&&&s    G2!H2
H 	
H	c                     / SQn / SQn[        U [        R                  " S5      S9n[        U[        R                  " S5      S9nX4X#4/n[        U5       H  u  nu  p[	        [
        XSS/S9n[	        [
        XS S9n[        SS	/U" S S
95        [        SU" SS
95        [        SU" SS
95        [        SU" SS
95        S H  nU" US
9U" US
9:w  a  M   e   M     g )N)rC   rC   r<   r   )rC   r   r   r   r   r   rC   r   rv   r   r   r   r   r   UUUUUU?r   r   )r   r   r   )r*   rH   rI   r   r   r#   r4   r3   )	r`   r_   r   r   rE   r   	recall_13
recall_allr   s	            ra   &test_precision_recall_f_ignored_labelsr   T  s     FF		!=J		!=Jz67D(FL&!QH	\6$G
!3*i.EFOYw-GH3Yz5RSGYw%?@ 6GW-G1LLLL 6  /rc   c            	      "   [         R                  " / SQ/ SQ/ SQ/ SQ/ SQ/ SQ/5      n [         R                  " / SQ/ SQ/ SQ/ S	Q/ S
Q/ SQ/5      nSn[        R                  " [        US9   [        XSS9  SSS5        g! , (       d  f       g= f)z:Test multiclass-multiouptut for `average_precision_score`.)r<   r<   rC   rC   r<   r   r   rC   r<   rC   r<   r   rC   ffffff?皙?皙?皙?333333?r   r   r   r   r   r   r   )r   r   r   )r   r   r   z.multiclass-multioutput format is not supportedmatchr<   	pos_labelN)rH   r   r   r   r   r   )r`   y_scoreerr_msgs      ra   -test_average_precision_score_non_binary_classr   k  sy    XX	
	F hh	
	G ?G	z	11= 
2	1	1s   ,B  
Bzy_true, y_scorer   r   rC   r<   r   r   r   r   )r   r   r   r   rC   rC   rC   rC   rC   rC   rC   )r   r   r   r   r   333333?r   rm   rm   rC   rC   c                 &    [        X5      S:X  d   eg)a  
Duplicate values with precision-recall require a different
processing than when computing the AUC of a ROC, because the
precision-recall curve is a decreasing curve
The following situation corresponds to a perfect
test statistic, the average_precision_score should be 1.
rC   Nr   r`   r   s     ra   -test_average_precision_score_duplicate_valuesr     s    8 #63q888rc   )r<   r<   rC   rC   r   )r   r   r   )r   r   r   r   )r   r   r   c                 &    [        X5      S:w  d   eg )Nr   r   r   s     ra   (test_average_precision_score_tied_valuesr     s    : #63s:::rc   c                      Sn [         R                  " [        U S9   [        / SQ/ SQSSS9  S S S 5        g ! , (       d  f       g = f)NzNote that pos_label \(set to 2\) is ignored when average != 'binary' \(got 'macro'\). You may use labels=\[pos_label\] to specify a single positive class.r   r   rC   r<   r<   r<   r   r   r   )r   warnsUserWarningr!   r   s    ra   (test_precision_recall_f_unused_pos_labelr    s9    
	  
k	-'yAw	
 
.	-	-	   6
Ac            	          [        SS9u  pnS nU" X5        U" U  Vs/ s H  n[        U5      PM     snU Vs/ s H  n[        U5      PM     sn5        g s  snf s  snf )NTr   c                    [        X5      n[        USS/SS//5        UR                  5       u  p4pVX6-  XE-  -
  n[        R                  " X4-   X5-   -  Xd-   -  Xe-   -  5      nUS:X  a  SOXx-  n	[        X5      n
[        XSS9  [        U
SSS9  g )	N   r         r   r<   decimal=
ףp=?)r   r5   flattenrH   sqrtr   r4   )r`   r_   cmtpfpfntnnumdentrue_mccmccs              ra   test*test_confusion_matrix_binary.<locals>.test  s    f-2Q!R12gggrw27+rw727CDq1ci/!#;!#tQ7rc   rb   r   r`   r_   r~   r  rW   s        ra   test_confusion_matrix_binaryr    sV    't4FA8 	&	!&Q#a&&	!F#;FqCFF#;<	!#;
   AA
c            	          [        SS9u  pnS nU" X5        U" U  Vs/ s H  n[        U5      PM     snU Vs/ s H  n[        U5      PM     sn5        g s  snf s  snf )NTr   c                 N    [        X5      n[        USS/SS//SS/SS///5        g )Nr  r
  r   r	  r    r5   )r`   r_   r  s      ra   r  5test_multilabel_confusion_matrix_binary.<locals>.test  s5    (82"a1b' 2b!Wq"g4FGHrc   r  r  s        ra   'test_multilabel_confusion_matrix_binaryr$    sW    't4FAI 	&	!&Q#a&&	!F#;FqCFF#;<	!#;r  c            	          [        SS9u  pnSS jnU" X5        U" U  Vs/ s H  n[        U5      PM     snU Vs/ s H  n[        U5      PM     snSS9  g s  snf s  snf )NFr   c           	      R   [        X5      n[        USS/SS//SS/SS//S	S
/SS///5        U(       a  / SQO/ SQn[        XUS9n[        USS/SS//S	S
/SS//SS/SS///5        U(       a  / SQO/ SQn[        XUS9n[        USS/SS//S	S
/SS//SS/SS//SS/SS///5        g )N/   r   r      &   r      r      r   r<      )021r   r<   rC   r   )r-  r.  r/  3)r   r<   rC   r   rp   r   r"  )r`   r_   string_typer  rv   s        ra   r  9test_multilabel_confusion_matrix_multiclass.<locals>.test  s   (82q'Ar7#r1gAw%72r(QG9LM	

 %0Y(G2q'Ar7#r2hB%8B7RG:LM	

 *5%,(Ga1b'"bAr7#a2q'"a1a&!		
rc   T)r2  )Fr  r  s        ra   +test_multilabel_confusion_matrix_multiclassr4    sX    'u5FA
6 	&	!&Q#a&&	!F#;FqCFF#;N	!#;s
   AA
csc_containercsr_containerc                    [         R                  " / SQ/ SQ/ SQ/5      n[         R                  " / SQ/ SQ/ SQ/5      nU" U5      nU" U5      nU " U5      nU " U5      n[         R                  " / SQ5      nSS	/SS//SS	/SS//S	S
/SS	///n	X$U/n
X5U/nU
 H"  nU H  n[        X5      n[        X5        M     M$     [        X#SS9n[        USS	/SS//SS/S	S//S	S/S
S	///5        [        X#S
S	/S9n[        US	S
/SS	//SS	/SS///5        [        X#S
S	/SS9n[        US	S	/SS//SS/S	S	//S	S/SS	///5        [        X#USS9n[        US
S	/S
S
//SS/S	S//S	S/SS	///5        g )Nr   r   rC   r   rC   rC   r   r   r   r   )r<   rC   r   rC   r   r<   T
samplewiser   )rv   r;  )sample_weightr;  r   r   )rH   r   r    r5   )r5  r6  r`   r_   
y_true_csr
y_pred_csr
y_true_csc
y_pred_cscr<  real_cmtruespreds
y_true_tmp
y_pred_tmpr  s                  ra   +test_multilabel_confusion_matrix_multilabelrF    s   
 XXy)Y78FXXy)Y78Fv&Jv&Jv&Jv&J HHY'MAA1a&1a&!1QFQF3CDG,E,E
J,ZDBr+    
%V	EBraVaV,1v1v.>!Q!Q@PQR 
%VQF	CBraVaV,1v1v.>?@ 
%VQFt	TBraVaV,1v1v.>!Q!Q@PQR 
%m
B raVaV,1v1v.>!Q!Q@PQRrc   c            	         [         R                  " / SQ/ SQ/ SQ/5      n [         R                  " / SQ/ SQ/ SQ/5      n[        R                  " [        SS9   [        XS	S
/S9  S S S 5        [        R                  " [        SS9   [        X/ SQ/ SQ/ SQ/S9  S S S 5        Sn[        R                  " [        US9   [        XS/S9  S S S 5        Sn[        R                  " [        US9   [        XS/S9  S S S 5        [        R                  " [        SS9   [        / SQ/ SQSS9  S S S 5        Sn[        R                  " [        US9   [        / SQ/ SQ// SQ/ SQ/5        S S S 5        g ! , (       d  f       GN= f! , (       d  f       N= f! , (       d  f       N= f! , (       d  f       N= f! , (       d  f       N= f! , (       d  f       g = f)Nr   r8  r9  r   r   r   inconsistent numbers of samplesr   rC   r<   r<  z)Sample weights must be 1D array or scalarrC   r<   r   )r<   r   r   )r   r   r   z%All labels must be in \[0, n labels\)r   r   r   zSamplewise metricsr   r   Tr:  z'multiclass-multioutput is not supported)r<   rC   r   rC   r   r<   )rH   r   r   r   r   r    )r`   r_   r   s      ra   'test_multilabel_confusion_matrix_errorsrL  C  sL   XXy)Y78FXXy)Y78F 
z)J	K#F1a&I 
L	z)T	U#9i*K	
 
V 7G	z	1#FB4@ 
26G	z	1#FA3? 
2 
z)=	>#IyTJ 
? 8G	z	1#Y	$:Y	<RS 
2	1+ 
L	K	U	U 
2	1 
2	1 
?	>
 
2	1sH   E.F =F,F"F3G.
E= 
F
F"
F03
G
Gz%normalize, cm_dtype, expected_results))truer   TUU?)predr   rN  )allr   geq?)Nr   r<   c                     / SQS-  n[        [        [        / SQ5      6 5      n[        X4U S9n[	        XR5        UR
                  R                  U:X  d   eg )Nr   r   	normalize)listr   r   r   r2   dtypekind)rS  cm_dtypeexpected_resultsy_testr_   r  s         ra   test_confusion_matrix_normalizerZ  a  sK     ]F%i012F	&I	>BB)88==H$$$rc   c                      / SQn / SQn[        XSS9nUR                  5       [        R                  " S5      :X  d   e[        R
                  " 5          [        R                  " S[        5        [        XSS9nS S S 5        WR                  5       [        R                  " S5      :X  d   e[        R
                  " 5          [        R                  " S[        5        [        XSS9  S S S 5        g ! , (       d  f       N|= f! , (       d  f       g = f)	N)r   r   r   r   rC   rC   rC   rC   )r   r   r   r   r   r   r   r   rM  rR         @r   rO  r   )r   sumr   r   r   r   r   RuntimeWarning)rY  r_   cm_truecm_preds       ra   ,test_confusion_matrix_normalize_single_classra  r  s    %F%Fv@G;;=FMM#.... 
	 	 	"g~6"6VD 
# ;;=FMM#....		 	 	"g~66: 
#	" 
#	" 
#	"s   &C.?&C?.
C<?
Dc                      / SQn / SQn[         R                  " [        SS9   [        X5        SSS5        g! , (       d  f       g= f)z8Test `confusion_matrix` warns when only one label found.r   r   r   r   zA single label was found inr   N)r   r  r  r   )rY  r_   s     ra   "test_confusion_matrix_single_labelrd    s0    FF	k)F	G( 
H	G	Gs	   7
Azparams, warn_msg)rC   rC   rC   r   r   r   r`   r_   z?`positive_likelihood_ratio` is ill-defined and set to `np.nan`.)r   r   r   r   r   r   zdNo samples were predicted for the positive class and `positive_likelihood_ratio` is set to `np.nan`.r   r   r   rC   rC   rC   z?`negative_likelihood_ratio` is ill-defined and set to `np.nan`.z9No samples of the positive class are present in `y_true`.c                     [         R                  " [        US9   [        S0 U D6  S S S 5        g ! , (       d  f       g = fNr   r   )r   r  r  r   )paramswarn_msgs     ra   test_likelihood_ratios_warningsrk    s*    X 
k	2)&) 
3	2	2   /
=zparams, err_msg)r   rC   r   rC   r   rC   rC   r   r   r<   zeclass_likelihood_ratios only supports binary classification problems, got targets of type: multiclassc                     [         R                  " [        US9   [        S0 U D6  S S S 5        g ! , (       d  f       g = frh  )r   r   r   r   )ri  r   s     ra   test_likelihood_ratios_errorsro    s)    $ 
z	1)&) 
2	1	1rl  c                     [         R                  " S/S-  S/S-  -   5      n [         R                  " S/S-  S/S-  -   S/S-  -   5      n[        X5      u  p#[        US5        [        US	5        [        X 5      u  p#[	        U[         R
                  S-  5        [        U[         R                  " S5      S
S9  [         R                  " S/S-  S/S-  -   5      n[        XUS9u  p#[        US5        [        US5        g )NrC   r   r   r  r<   
   r
  g?g_B{	%?g-q=)rtolr      r   r   rI  gUUUUUU@gqq?)rH   r   r   r2   r5   nanr   )r`   r_   posnegr<  s        ra   test_likelihood_ratiosrw    s     XXqcAgb()FXXqcAgb(A3723F&v6HCC!C! 'v6HCsBFFQJ'C!51
 HHcURZ3%!)34M&v]SHCC C!rc   raise_warningc                     [         R                  " SS/5      n[         R                  " SS/5      nSn[        R                  " [        US9   [        XU S9  SSS5        g! , (       d  f       g= f)z]Test that class_likelihood_ratios raises a `FutureWarning` when `raise_warning`
param is set.rC   r   zI`raise_warning` was deprecated in version 1.7 and will be removed in 1.9.r   )rx  N)rH   r   r   r  FutureWarningr   )rx  r`   r_   r   s       ra   0test_likelihood_ratios_raise_warning_deprecationr{    sQ     XXq!fFXXq!fF
UC	m3	/mL 
0	/	/   A  
A.c                  d   [         R                  " / SQ5      n [         R                  " / SQ5      n[        XSS9u  p#U[        R                  " S5      :X  d   e[         R                  " / SQ5      n [         R                  " / SQ5      n[        XSS9u  p4U[        R                  " S5      :X  d   eg)z{Test that class_likelihood_ratios returns the worst scores `1.0` for both LR+ and
LR- when `replace_undefined_by=1` is set.r9  r   rC   replace_undefined_byr   r   N)rH   r   r   r   r   )r`   r_   positive_likelihood_ratior~   negative_likelihood_ratios        ra   1test_likelihood_ratios_replace_undefined_by_worstr    s    
 XXi FXXi F#:Q$  %c(:::: XXi FXXi F#:Q$ A %c(::::rc   r  LR+r   LR-g      )r  r  r   rt  r\  c                     [         R                  " SS/5      n[         R                  " SS/5      nSn[        R                  " [        US9   [        XU S9  SSS5        g! , (       d  f       g= f)zTest that class_likelihood_ratios raises a `ValueError` if the input dict for
`replace_undefined_by` is in the wrong format or contains impossible values.rC   r   zGThe dictionary passed as `replace_undefined_by` needs to be in the formr   r~  N)rH   r   r   r   r   r   )r  r`   r_   r   s       ra   6test_likelihood_ratios_wrong_dict_replace_undefined_byr    sU     XXq!fFXXq!fF
SC	z	-1E	
 
.	-	-r|  zreplace_undefined_by, expectedc                 &   [         R                  " / SQ5      n[         R                  " / SQ5      n[        X#U S9u  pE[         R                  " U5      (       a  [         R                  " U5      (       d   egU[        R
                  " U5      :X  d   eg)z|Test that the `replace_undefined_by` param returns the right value for the
positive_likelihood_ratio as defined by the user.r9  r   r~  NrH   r   r   isnanr   r   )r  expectedr`   r_   r  r~   s         ra   0test_likelihood_ratios_replace_undefined_by_0_fpr  *  sr     XXi FXXi F#:-A$  
xxxx12222(FMM(,CCCCrc   r   c                 &   [         R                  " / SQ5      n[         R                  " / SQ5      n[        X#U S9u  pE[         R                  " U5      (       a  [         R                  " U5      (       d   egU[        R
                  " U5      :X  d   eg)z|Test that the `replace_undefined_by` param returns the right value for the
negative_likelihood_ratio as defined by the user.r   r   r~  Nr  )r  r  r`   r_   r~   r  s         ra   0test_likelihood_ratios_replace_undefined_by_0_tnr  F  sq     XXi FXXi F#:-A$ A 
xxxx12222(FMM(,CCCCrc   c                     [         R                  " S/S-  S/S-  -   5      n [         R                  " S/S-  S/S-  -   S/S-  -   S/S-  -   5      n[        X5      n[        USS	S
9  U[        X5      :X  d   e[         R                  " U S/S-  5      n [         R                  " US/S-  5      n[        XSS/S9U:X  d   e[        [        X 5      S5        [         R                  " S/S-  S/S-  -   S/S-  -   5      n [         R                  " S/S-  S/S-  -   S/S-  -   5      n[        [        X5      SSS
9  [         R                  " S/S-  S/S-  -   S/S-  -   5      n [         R                  " S/S-  S/S-  -   S/S-  -   5      n[        [        X5      SSS
9  [        [        XSS9SSS
9  [        [        XSS9SSS
9  g )Nr   (   rC   <   ro   rq  2   gʡE?r   r  r<   r   r   r   .   ,   4          g??g+?r?   weightsg_vO?	quadraticg#?)rH   r   r   r3   append)r   r   kappas      ra   test_cohen_kappar  b  s    
1#(aS2X%	&B	1#(aS2X%b0A38;	<Bb%Eua0%b---- 
2sQw	B	2sQw	BRQF3u<<<)"137 
1#(aS2X%b0	1B	1#(aS2X%b0	1B)"161E 
1#(aS2X%b0	1B	1#(aS2X%b0	1B)"161E)"(CVUVW"+6rc   c                      SS/n [         R                  " S/S-  S/S-  -   5      n[         R                  " S/S-  5      n[        R                  " [        SS9   [        XU S	9  S
S
S
5        g
! , (       d  f       g
= f)zLTest that correct error is raised when users pass labels that are not in y1.rC   r<   r   r   r   rq  z6At least one label in `labels` must be present in `y1`r   r   N)rH   r   r   r   r   r   )rv   r   r   s      ra   (test_cohen_kappa_score_error_wrong_labelr    sl    VF	3%!)seai'	(B	3%"*	B	R
 	"0
 
 
s   A--
A;zy_true, y_predr   r   c                    [         R                  " 5          [         R                  " S5        U " XUS9nSSS5        [        R                  " U5      (       a  [        R                  " W5      (       d   egWU:X  d   eg! , (       d  f       NP= f)zeCheck the behaviour of `zero_division` when setting to 0, 1 or np.nan.
No warnings should be raised.
r   r   N)r   r   r   rH   r  )r   r`   r_   r   results        ra   !test_zero_division_nan_no_warningr    sl     
	 	 	"g&mD 
# 
xxxx&&& 
#	"s   A>>
Bc                     [         R                  " [        5         U " XSS9nSSS5        WS:X  d   eg! , (       d  f       N= f)zlCheck the behaviour of `zero_division` when setting to "warn".
A `UndefinedMetricWarning` should be raised.
r   r  Nr   )r   r  r   )r   r`   r_   r  s       ra   test_zero_division_nan_warningr    s7     
,	-f= 
.S== 
.	-s	   4
Ac                     [         R                  R                  U 5      nUR                  SSSS9nUR                  SSSS9n[	        [        X#5      [         R                  " X#5      S   S5        g )Nr   r<   ro   sizer   rC   rq  )rH   rL   rM   randintr3   r   corrcoef)global_random_seedr[   r`   r_   s       ra   -test_matthews_corrcoef_against_numpy_corrcoefr    sa    
))

 2
3C[[AB['F[[AB['F&)2;;v+Ft+Lbrc   c                    [         R                  R                  U 5      nUR                  SSSS9nUR                  SSSS9nUR	                  S5      n[        X#US9n[        U5      n[        [        U5       VVV	s/ s HD  n[        U5        H1  n[        U5        H  n	XWU4   XXU	4   -  XYU4   XWU4   -  -
  PM      M3     MF     sn	nn5      n
[        [        U5       VVVs/ s Hk  nUS S 2U4   R                  5       [         R                  " [        U5       VVs/ s H#  n[        U5        H  oU:w  d  M
  X\U4   PM     M%     snn5      -  PMm     snnn5      n[         R                  " [        U5       VVVs/ s Hj  nXWS S 24   R                  5       [         R                  " [        U5       VVs/ s H#  n[        U5        H  oU:w  d  M
  X[U4   PM     M%     snn5      -  PMl     snnn5      nU
[         R                  " X-  5      -  n[        X#US9n[        UUS5        g s  sn	nnf s  snnf s  snnnf s  snnf s  snnnf )Nr   r<   ro   r  rI  rq  )rH   rL   rM   r  randr   ry   r]  ranger  r   r3   )r  r[   r`   r_   r<  CNkmlcov_ytypr   gcov_ytytcov_ypyp
mcc_jurmanmcc_ourss                    ra   %test_matthews_corrcoef_against_jurmanr    s    ))

 2
3C[[AB['F[[AB['FHHRLM}EAAA 1X	
1X1X dGa1gQ$!qD' 11  2 2	
H  1X	
  adGKKMffuQxLx!qA!Vga1ggxLMN	
H vv 1X	
  dGKKMffuQxLx!qA!Vga1ggxLMN	
H BGGH$788J }MH*b11	
 M	
 M	
sC   <AH+$:H8H2:H2H8>9I7H?H?!I2H8?Ic           	         [         R                  R                  U 5      nUR                  SSSS9 Vs/ s H  o"S:X  a  SOSPM     nn[	        [        X35      S5        U Vs/ s H  o"S:X  a  SOSPM     nn[	        [        X45      S5        [        USS/S	9n[         R                  " USS5      n[	        [        X55      S5        [	        [        / S
Q/ S
Q5      S5        [	        [        US/[        U5      -  5      S5        / SQn/ SQn[	        [        Xg5      S5        S/S-  S/S-  -   n[        R                  " [        5         [	        [        XgUS9S5        S S S 5        g s  snf s  snf ! , (       d  f       g = f)Nr   r<   ro   r  r   r   r   r   r   rc  r   )rC   r   rC   rC   r   rC   rC   rC   r   rC   rC   rC   rC   rC   rC   rC   r   rC   rC   rC   )rC   rC   rC   r   r   rC   rC   rC   rC   r   rC   rC   rC   r   rC   rC   rC   r   rC   rC   rC   rq  rI  )rH   rL   rM   r  r3   r   r*   wherery   r   r   AssertionError)	r  r[   r   r`   
y_true_invy_true_inv2y_1y_2masks	            ra   test_matthews_corrcoefr    s\   
))

 2
3C.1kk!QRk.HI.H!Vc$.HFI )&93? 5;;Fqc#s*FJ;)&=rB #s<K((;S1K)&>C ),EsK )&3%#f+2EFL GC
FC)#3S9 38qcBhD 
~	&-cdKSQ 
'	&; J <. 
'	&s   E#!E(E--
E;c                    [         R                  R                  U 5      n[        S5      nSnUR	                  SUSS9 Vs/ s H  n[        X$-   5      PM     nn[        [        XU5      S5        / SQn/ SQn[        [        XV5      S	5        / SQn/ S
Qn[        [        XW5      S[         R                  " S5      -  5        / SQn/ SQn[        [        XX5      S5        / SQn/ SQn[        [        XX5      S5        / SQn	/ SQn
[        [        X5      S5        / SQn/ SQn/ SQn[        [        XXUS9S5        / SQn/ SQn/ SQn[        [        XXUS9S5        g s  snf )Nr   r   r   ro   r  r   )r   r   rC   rC   r<   r<   )r<   r<   r   r   rC   rC   g      )rC   rC   r   r   r   r   ii  r   )r   r   r   r   	r   rC   r<   r   rC   r<   r   rC   r<   )	rC   rC   rC   r<   r<   r<   r   r   r   )r   r   rC   rC   r<   rm  rC   rC   rC   rC   r   rI  r   r   rC   rC   r   r   )	rH   rL   rM   ordr  chrr3   r   r  )r  r[   ord_a	n_classesr   r`   
y_pred_bad
y_pred_minr_   r  r  r<  s               ra   !test_matthews_corrcoef_multiclassr    sM   
))

 2
3CHEI&)kk!YRk&HI&Hc%)n&HFI )&93?  F#J)&=tD  F#J)&=sRWWWEU?UV FF)&93? FF)&93? &C
%C)#3S9 FF#M&F FF M&F_ Js    En_pointsd   i'  c                   ^ [         R                  R                  U5      mS nU4S jn[         R                  " SS/U 5      n[	        [        XD5      S5        [         R                  " / SQU 5      n[	        [        XD5      S5        U" U 5      u  pV[	        [        XU5      S5        [	        [        XV5      U" XV5      5        g )Nc                     [        X5      nUS   nUS   nUS   n[        U 5      nX5-   U-  nX4-   U-  nX6-  Xx-  -
  n	X-  SU-
  -  SU-
  -  n
U	[        R                  " U
5      -  $ )NrC   rC   )rC   r   r  rC   )r   ry   rH   r  )r`   r_   conf_matrixtrue_pos	false_pos	false_negr  pos_rateactivitymcc_numeratormcc_denominators              ra   mcc_safe1test_matthews_corrcoef_overflow.<locals>.mcc_safeI  s    &v6t$%	%	v;(H4(H4 +h.AA"-X>!h,Orww777rc   c                 t   > TR                  U 5      nUSTR                  U 5      S-
  -  -   nUS:  nUS:  nX44$ )Nr   r   )random_sample)r  x_truex_predr`   r_   r[   s        ra   	random_ys2test_matthews_corrcoef_overflow.<locals>.random_ysU  sL    ""8,#!2!28!<s!BCC##~rc   r   r   )r   r   r\  )rH   rL   rM   repeatr3   r   )r  r  r  r  arrr`   r_   r[   s          @ra   test_matthews_corrcoef_overflowr  D  s     ))

 2
3C
8 ))S#J
)C)#3S9
))OX
.C)#3S9x(NF)&93?)&98F;STrc   c                     [        SS9u  pn[        XS S9u  p4pV[        U/ SQS5        [        U/ SQS5        [        U/ SQS5        [        U/ SQ5        [	        XS	S
S9n[        USS5        [        XS
S9n[        USS5        [        XS
S9n	[        U	SS5        [	        XSS9n[        USS5        [        XSS9n[        USS5        [        XSS9n	[        U	SS5        [	        XSS9n[        USS5        [        XSS9n[        USS5        [        XSS9n	[        U	SS5        [        R                  " [        5         [	        XSS9  S S S 5        [        R                  " [        5         [        XSS9  S S S 5        [        R                  " [        5         [        XSS9  S S S 5        [        R                  " [        5         [        XSSS9  S S S 5        [        X/ SQS S9u  p4pV[        U/ SQS5        [        U/ SQS5        [        U/ SQS5        [        U/ SQ5        g ! , (       d  f       N= f! , (       d  f       N= f! , (       d  f       N= f! , (       d  f       N= f)NFr   r   )(\?Q?gzG?r<   )HzG?g
ףp=
?rm   )Q?333333?r  )rf   rl   ro   rC   r   r  g(\?r   r   gRQ?r   gGz?r   r   r   r   r0  r   )r  g=
ףp=?r  )r  rm   r   )r  r  r  )rf   ro   rl   )rb   r!   r4   r5   r"   r#   r   r   r   r   r   )
r`   r_   r~   rZ   r   r   r   r   r   r   s
             ra   )test_precision_recall_f1_score_multiclassr  f  s   'u5FA 1NJA!a!3Q7a!3Q7a!3Q7q,' 
1g	FBb$*	fg	6Bb$*	&'	2Bb$*		9Bb$*	fg	6Bb$*	&'	2Bb$*		<Bb$*	fj	9Bb$*	&*	5Bb$*	z	"	: 
#	z	"VY7 
#	z	"3 
#	z	"FIC@ 
# 1y$JA! a!3Q7a!3Q7a!3Q7q,'! 
#	"	"	"	"	"	"	"s0   ?H&,H7II&
H47
I
I
I'r   )r   r   r   r   Nc                     [         R                  " / SQ/5      n[         R                  " / SQ/5      n[        X/ SQ/ U S9u  p4pV[        US5        [        US5        [        US5        U c  [        U/ SQ5        g g )Nr  r   r   rC   rC   )r   r   rC   r<   )rv   warn_forr   r   r   rC   rC   r   )rH   r   r!   r5   )r   r`   r_   rZ   r   r   r   s          ra   ;test_precision_refcall_f1_score_multilabel_unordered_labelsr    so     XX|n%FXX|n%F0|b'JA! q!q!q!1l+ rc   c                  "   [         R                  " / SQ5      n [         R                  " / SQ5      n[        XS S9u  p#pE[        XSS9u  pgpU[         R                  " U5      :X  d   eU[         R                  " U5      :X  d   eU[         R                  " U5      :X  d   e[        XSS9u  pgp[         R                  " U 5      n	U[         R
                  " X)S9:X  d   eU[         R
                  " X9S9:X  d   eU[         R
                  " XIS9:X  d   eg )N)r   rC   r   r   rC   rC   r   rC   r   r   rC   r   rC   r   rC   )rC   rC   r   rC   r   rC   rC   rC   rC   r   rC   r   rC   r   rC   r   r   r   r  )rH   r   r!   r   bincountr   )
r`   r_   r   r   r   r~   rZ   r   r   rj   s
             ra   .test_precision_recall_f1_score_binary_averagedr    s    XXCDFXXCDF 4FDQMBB0QJA!0TJA!kk&!G

2////

2////

2////rc   c                  n   [         R                  " SS9n  [         R                  " / SQ5      n[         R                  " / SQ5      n[        [	        XSS9SS5        [        [        XSS9SS5        [        [        XSS9SS5        [         R                  " S	0 U D6  g ! [         R                  " S	0 U D6  f = f)
Nraise)rP  )r   rC   r<   r   rC   r<   )r<   r   rC   rC   r<   r   r   r   r   r<   r   )rH   seterrr   r3   r"   r#   r   )old_error_settingsr`   r_   s      ra   test_zero_precision_recallr    s     w/	(,-,-OFGLcSTULI3PQRHVWEsAN 			'&'		'&'s   A/B B4c                     [        SS9u  pn[        XSS/S9n[        USS/SS//5        [        XS	S/S9n[        US
S	/SS//5        [        R                  " U 5      S-   n[        XS	U/S9n[        US
S/SS//5        g )NFr   r   rC   r   r(  r   r   r<   r,  rf   )rb   r   r5   rH   max)r`   r_   r~   r  extra_labels        ra   .test_confusion_matrix_multiclass_subset_labelsr     s    'u5FA 
&!Q	8BrRGaV,- 
&!Q	8BrRGb!W-. &&.1$K	&![1A	BBrRGaV,-rc   zlabels, err_msgz+'labels' should contain at least one label.r   r   z.At least one label specified must be in y_truez
empty listzunknown labels)idsc                     [        SS9u  p#n[        R                  " [        US9   [	        X#U S9  S S S 5        g ! , (       d  f       g = f)NFr   r   r   )rb   r   r   r   r   )rv   r   r`   r_   r~   s        ra   test_confusion_matrix_errorr    s7     (u5FA	z	17 
2	1	1s	   :
Ac            
      z   / SQn [         R                  " [        U 5      5      n[        X 5      nUR                  [         R
                  :X  d   e[         R                  [         R                  [         R                  4 H;  n[        X UR                  USS9S9nUR                  [         R
                  :X  a  M;   e   [         R                  [         R                  S [        4 H;  n[        X UR                  USS9S9nUR                  [         R                  :X  a  M;   e   [         R                  " [        U 5      S[         R                  S9n[        X US9nUS   S:X  d   eUS   S	:X  d   e[         R                  " [        U 5      S
[         R
                  S9n[        X US9nUS   S
:X  d   eUS   S:X  d   eg )Nr   F)copyrI  l    rU  r   r   r  l    l    )rH   onesry   r   rU  int64bool_int32uint64astypefloat32float64objectfulluint32)rW   weightr  rU  s       ra   test_confusion_matrix_dtyper    sh   AWWSV_F	!	B88rxx((BHHbii0a&--E-2RSxx288### 1 **bjj$7a&--E-2RSxx2::%%% 8
 WWSVZryy9F	!f	5Bd8z!!!d8z!!! WWSV0AF	!f	5Bd8****d8r>>rc   rU  )Int64Float64booleanc                     [         R                  " S5      n[        R                  " / SQ5      nUR	                  X S9nUR	                  / SQSS9n[        X45      n[        X$5      n[        XV5        g)zcChecks that confusion_matrix works with pandas nullable dtypes.

Non-regression test for gh-25635.
pandas)	rC   r   r   rC   r   rC   rC   r   rC   r  )	r   r   rC   rC   r   rC   rC   rC   rC   r
  N)r   importorskiprH   r   Seriesr   r5   )rU  pd	y_ndarrayr`   y_predictedoutputexpected_outputs          ra   %test_confusion_matrix_pandas_nullabler"    sb     
		X	&B45IYYyY.F))7w)GKf2F&y>Ov/rc   c            	          [         R                  " 5       n [        U SS9u  pnSn[        UU[        R
                  " [        U R                  5      5      U R                  S9nXT:X  d   eg )NFre   a|                precision    recall  f1-score   support

      setosa       0.83      0.79      0.81        24
  versicolor       0.33      0.10      0.15        31
   virginica       0.42      0.90      0.57        20

    accuracy                           0.53        75
   macro avg       0.53      0.60      0.51        75
weighted avg       0.51      0.53      0.47        75
rv   rw   r   rD   rb   r   rH   rI   ry   rw   r}   r`   r_   r~   r   r   s         ra   %test_classification_report_multiclassr'    sh    D'UCFA
O #yyT../0&&	F $$$rc   c                  :    / SQ/ SQpSn[        X5      nX2:X  d   eg )N)	r   r   r   rC   rC   rC   r<   r<   r<   r  a|                precision    recall  f1-score   support

           0       0.33      0.33      0.33         3
           1       0.33      0.33      0.33         3
           2       0.33      0.33      0.33         3

    accuracy                           0.33         9
   macro avg       0.33      0.33      0.33         9
weighted avg       0.33      0.33      0.33         9
r   )r`   r_   r   r   s       ra   .test_classification_report_multiclass_balancedr)  9  s)    02MF
O #62F$$$rc   c                  p    [         R                  " 5       n [        U SS9u  pnSn[        X5      nXT:X  d   eg )NFre   a|                precision    recall  f1-score   support

           0       0.83      0.79      0.81        24
           1       0.33      0.10      0.15        31
           2       0.42      0.90      0.57        20

    accuracy                           0.53        75
   macro avg       0.53      0.60      0.51        75
weighted avg       0.51      0.53      0.47        75
)r   rD   rb   r   r&  s         ra   :test_classification_report_multiclass_with_label_detectionr+  K  s@    D'UCFA
O #62F$$$rc   c            	          [         R                  " 5       n [        U SS9u  pnSn[        UU[        R
                  " [        U R                  5      5      U R                  SS9nXT:X  d   eg )NFre   a|                precision    recall  f1-score   support

      setosa    0.82609   0.79167   0.80851        24
  versicolor    0.33333   0.09677   0.15000        31
   virginica    0.41860   0.90000   0.57143        20

    accuracy                        0.53333        75
   macro avg    0.52601   0.59615   0.50998        75
weighted avg    0.51375   0.53333   0.47310        75
r   )rv   rw   digitsr%  r&  s         ra   1test_classification_report_multiclass_with_digitsr.  _  sk    D'UCFA
O #yyT../0&&F $$$rc   c                      [        SS9u  pn[        R                  " / SQ5      U    n [        R                  " / SQ5      U   nSn[        X5      nXC:X  d   eSn[        X/ SQS9nXC:X  d   eg )NFr   )bluegreenreda|                precision    recall  f1-score   support

        blue       0.83      0.79      0.81        24
       green       0.33      0.10      0.15        31
         red       0.42      0.90      0.57        20

    accuracy                           0.53        75
   macro avg       0.53      0.60      0.51        75
weighted avg       0.51      0.53      0.47        75
a|                precision    recall  f1-score   support

           a       0.83      0.79      0.81        24
           b       0.33      0.10      0.15        31
           c       0.42      0.90      0.57        20

    accuracy                           0.53        75
   macro avg       0.53      0.60      0.51        75
weighted avg       0.51      0.53      0.47        75
r   rw   rb   rH   r   r   )r`   r_   r~   r   r   s        ra   7test_classification_report_multiclass_with_string_labelr5  z  sy    'u5FAXX./7FXX./7F
O #62F$$$
O #6PF$$$rc   c                      [        SS9u  pn[        R                  " / SQ5      nX0   n X1   nSn[        X5      nXT:X  d   eg )NFr   )u   blue¢u   green¢u   red¢u                precision    recall  f1-score   support

       blue¢       0.83      0.79      0.81        24
      green¢       0.33      0.10      0.15        31
        red¢       0.42      0.90      0.57        20

    accuracy                           0.53        75
   macro avg       0.53      0.60      0.51        75
weighted avg       0.51      0.53      0.47        75
r4  r`   r_   r~   rv   r   r   s         ra   8test_classification_report_multiclass_with_unicode_labelr8    sM    'u5FAXX:;F^F^F
O #62F$$$rc   c                      [        SS9u  pn[        R                  " / SQ5      nX0   n X1   nSn[        X5      nXT:X  d   eg )NFr   )r0  greengreengreengreengreenr2  a                             precision    recall  f1-score   support

                     blue       0.83      0.79      0.81        24
greengreengreengreengreen       0.33      0.10      0.15        31
                      red       0.42      0.90      0.57        20

                 accuracy                           0.53        75
                macro avg       0.53      0.60      0.51        75
             weighted avg       0.51      0.53      0.47        75
r4  r7  s         ra   <test_classification_report_multiclass_with_long_string_labelr;    sM    'u5FAXX23F^F^F
O #62F$$$rc   c                      / SQn / SQn/ SQnSn[         R                  " [        US9   [        XSS/US9  S S S 5        g ! , (       d  f       g = f)	Nr   r   r<   r   r   r   r<   r<   r   r   zclass 0zclass 1zclass 2z6labels size, 2, does not match size of target_names, 3r   r   r<   r$  )r   r  r  r   )r`   r_   rw   r   s       ra   =test_classification_report_labels_target_names_unequal_lengthr@    s@    FF4L
BC	k	-faV,W 
.	-	-s	   ?
Ac                      / SQn / SQn/ SQnSn[         R                  " [        US9   [        XUS9  S S S 5        g ! , (       d  f       g = f)Nr=  r>  r?  zaNumber of classes, 2, does not match size of target_names, 3. Try specifying the labels parameterr   r3  )r   r   r   r   )r`   r_   rw   r   s       ra   @test_classification_report_no_labels_target_names_unequal_lengthrB    sA    FF4L	. 
 
z	1f<H 
2	1	1s	   <
A
c                  h    Sn Sn[        SXSS9u  p#[        SXSS9u  p$Sn[        X45      nXe:X  d   eg )Nr   r  rC   r   )rY   rX   r  rB   a                precision    recall  f1-score   support

           0       0.50      0.67      0.57        24
           1       0.51      0.74      0.61        27
           2       0.29      0.08      0.12        26
           3       0.52      0.56      0.54        27

   micro avg       0.50      0.51      0.50       104
   macro avg       0.45      0.51      0.46       104
weighted avg       0.45      0.51      0.46       104
 samples avg       0.46      0.42      0.40       104
)r   r   )r  rX   r~   r`   r_   r   r   s          ra   %test_multilabel_classification_reportrD    sS    II.	QIA /	QIAO #62F$$$rc   c                  2   [         R                  " / SQ/ SQ/5      n [         R                  " / SQ/ SQ/5      n[        X5      S:X  d   e[        X 5      S:X  d   e[        X5      S:X  d   e[        U[         R                  " U5      5      S:X  d   e[        U [         R                  " U 5      5      S:X  d   e[        U [         R                  " U R
                  5      5      S:X  d   e[        U[         R                  " U R
                  5      5      S:X  d   eg )Nr   r   r   r   r   rC   )rH   r   r$   r   r   rG   r   s     ra   $test_multilabel_zero_one_loss_subsetrF    s    	9i(	)B	9i(	)B C''' A%%% A%%%R^^B/0A555R^^B/0A555RXXbhh/0A555RXXbhh/0A555rc   c                      [         R                  " / SQ/ SQ/5      n [         R                  " / SQ/ SQ/5      n[         R                  " SS/5      n[        X5      S:X  d   e[        X 5      S:X  d   e[        X5      S:X  d   e[        USU-
  5      S:X  d   e[        U SU -
  5      S:X  d   e[        U [         R                  " U R                  5      5      S:X  d   e[        U[         R                  " U R                  5      5      S	:X  d   e[        XUS
9S:X  d   e[        U SU-
  US
9S:X  d   e[        U [         R
                  " U 5      US
9S:X  d   e[        U S   US   5      [        U S   US   5      :X  d   eg )Nr   r   r   rC   r   UUUUUU?r   r   r   rI  gUUUUUU?gUUUUUU?)rH   r   r   r   rG   
zeros_like
sp_hamming)r   r   ws      ra   test_multilabel_hamming_lossrL    se   	9i(	)B	9i(	)B
!QA5(((1$$$1$$$AF#q(((AF#q(((BHHRXX./5888BHHRXX./3666a0H<<<AF!4	AAABMM"-Q?7JJJ1r!u%BqE2a5)AAAArc   c                     [         R                  " / SQ5      n [         R                  " / SQ5      nSn[        R                  " [        US9   [        XSSS9  S S S 5        [         R                  " / SQ/ SQ/5      n [         R                  " / S	Q/ S
Q/5      nSn[        R                  " [        US9   [        XSSS9  S S S 5        [         R                  " / SQ5      n [         R                  " / SQ5      nSn[        R                  " [        US9   [        XSS9  S S S 5        Sn[        R                  " [        US9   [        XSS9  S S S 5        Sn[        R                  " [        US9   [        XSSS9  S S S 5        g ! , (       d  f       GN2= f! , (       d  f       N= f! , (       d  f       N= f! , (       d  f       Nr= f! , (       d  f       g = f)N)r   rC   r   rC   rC   z>pos_label=2 is not a valid label. It should be one of \[0, 1\]r   rU   r<   r   r   r   r   r   r   Target is multilabel-indicator but average='binary'. Please choose another average setting, one of \[None, 'micro', 'macro', 'weighted', 'samples'\].r   )r   rC   rC   r   r<   r  Target is multiclass but average='binary'. Please choose another average setting, one of \[None, 'micro', 'macro', 'weighted'\].r   zJSamplewise metrics are not available outside of multilabel classification.r   zNote that pos_label \(set to 3\) is ignored when average != 'binary' \(got 'micro'\). You may use labels=\[pos_label\] to specify a single positive class.r   r   )rH   r   r   r   r   r   r  r  )r`   r_   r   msg1msg2msg3r   s          ra   test_jaccard_score_validationrT  #  sZ   XXo&FXXo&FOG	z	1fh!D 
2 XXy),-FXXy),-F	6 	
 
z	.fh"E 
/ XXo&FXXo&F	 	
 
z	.fh7 
/WD	z	.fi8 
/	  
k	-fgC 
.	-A 
2	1 
/	. 
/	. 
/	. 
.	-s<   F3FF& F7.G
F
F#&
F47
G
Gc           	         [         R                  " / SQ/ SQ/5      n[         R                  " / SQ/ SQ/5      n[        XSS9S:X  d   e[        XSS9S:X  d   e[        X"SS9S:X  d   e[        U[         R                  " U5      SS9S:X  d   e[        U[         R                  " U5      SS9S:X  d   e[        U[         R                  " UR
                  5      SS9S:X  d   e[        U[         R                  " UR
                  5      SS9S:X  d   e[         R                  " / SQ/ S	Q/5      n[         R                  " / S
Q/ SQ/5      n[        [        X4SS9S5        [        [        X4SS9S5        [        [        X4SS9S5        [        [        X4SSS/S9S5        [        [        X4SSS/S9S5        [        [        X4S S9[         R                  " / SQ5      5        [         R                  " / SQ/ SQ/5      n[         R                  " / S
Q/ SQ/5      n[        [        X4SS9S5        [        [        X4SS9S5        Sn[        R                  " [        US9   [        X4S/SS9  S S S 5        Sn[        R                  " [        US9   [        X4S/SS9  S S S 5        Sn[        R                  " [        US9   [        [         R                  " SS//5      [         R                  " SS//5      SS9S:X  d   e S S S 5        Sn[        R                  " [        US9   [        [         R                  " SS/SS//5      [         R                  " SS/SS//5      SS9S:X  d   e S S S 5        [        U 5      (       a   eg ! , (       d  f       GN = f! , (       d  f       GN= f! , (       d  f       N= f! , (       d  f       NV= f)Nr   r   r   r   r   r   rC   r   r   r   r   r   r   r   g?r<   r   r   )r   r   r   r   r   g      ?z	Got 4 > 2r   r   r   z
Got -1 < 0r   zXJaccard is ill-defined and being set to 0.0 in labels with no true or predicted samples.zXJaccard is ill-defined and being set to 0.0 in samples with no true or predicted labels.)rH   r   r   r   r   rG   r3   r5   r   r   r   r  r   rT  )recwarnr   r   r`   r_   rR  rS  r   s           ra   test_multilabel_jaccard_scorerW  K  s;   	9i(	)B	9i(	)B
 3t;;;3q8883q888R^^B/CqHHHR^^B/CqHHHRXXbhh/CqHHHRXXbhh/CqHHHXXy),-FXXy),-FfgFPfgFPfiH(SfiAG fiAG fd3RXX>U5V XXy),-FXXy),-FfgFPfjI7SD	z	.faS'B 
/D	z	.fbT7C 
/	- 
 
,C	8"((QF8,bhhAx.@'R	
 
9	, 
 
,C	81a&1a&)*1a&1a&)*!
 	
 
9 G}}}A 
/	. 
/	. 
9	8 
9	8s2   M>N8A N"AN3>
N
N"
N03
Oc           
         / SQn/ SQn/ SQn[        5       nUR                  U5        UR                  U5      nUR                  U5      n[        [        X5      n[        [        XV5      nSS/SS/SS/S/S/S/S /n	SS/SS	/S	S/S/S/S	/S /n
S
 H+  n[        X5       H  u  p[        U" XS9U" XS95        M     M-     [        R                  " SS/SS/SS//5      n[        R                  " SS/SS/SS//5      n[        5          [	        XSS9S:X  d   e S S S 5        [        U 5      (       a   eg ! , (       d  f       N!= f)N)antrY  catrZ  rY  rZ  birdr[  )rZ  rY  rZ  rZ  rY  r[  r[  rZ  )rY  r[  rZ  rY  r[  rZ  r   rC   r<   )r   r   r   Nr   r   r   )r)   rQ   	transformr   r   zipr3   rH   r   r6   rT  )rV  r`   r_   rv   lbr   r   multi_jaccard_scorebin_jaccard_scoremulti_labels_listbin_labels_listr   m_labelb_labels                 ra   test_multiclass_jaccard_scorere    si   GFGF#F		BFF6Nf%Jf%J!-@zF						 1v1v1vsQC!dCO 8 #$5 GG#GD!'B !H 8 XX1v1v1v./FXX1v1v1v./F		VZ@AEEE 
 G}}} 
	s   D44
Ec           	         [        S/S/SS9S:X  d   eSn[        R                  " [        US9   [        SS/SS/SS9S:X  d   e S S S 5        [        S/S/SSS9S	:X  d   e[        R
                  " / S
Q5      n[        R
                  " / SQ5      n[        [        X#SS9S5        [        [        X#SSS9S5        [        U 5      (       a   eg ! , (       d  f       N= f)NrC   r   rU   r   r   zOJaccard is ill-defined and being set to 0.0 due to no true or predicted samplesr   r  r   )rC   r   rC   rC   r   )rC   r   rC   rC   rC   r   rN  r   )r   r   r  r   rH   r   r3   rT  )rV  r   r`   r_   s       ra   !test_average_binary_jaccard_scorerg    s    !qc84;;;	'  
,C	8aVaVX>#EEE 
9 !qcQASHHHXXo&FXXo&FfhGQfh!Dg G}}} 
9	8s   C
Cc                  .   [         R                  " / SQ/ SQ/5      n [         R                  " / SQ/ SQ/5      nSn[        R                  " [        US9   [        XSSS9nU[        R                  " S5      :X  d   e S S S 5        g ! , (       d  f       g = f)	Nr   r   r   r   zJaccard is ill-defined and being set to 0.0 in samples with no true or predicted labels. Use `zero_division` parameter to control this behavior.r   r   r   r   r   r   )rH   r   r   r  r   r   r   )r`   r_   r   scores       ra   (test_jaccard_score_zero_division_warningrl    sw     XXy),-FXXy),-F	C 
 
,C	8fivVc**** 
9	8	8s   (B
Bzzero_division, expected_scorer  )rC   r   c                 V   [         R                  " / SQ/ SQ/5      n[         R                  " / SQ/ SQ/5      n[        R                  " 5          [        R                  " S[
        5        [        X#SU S9nS S S 5        W[        R                  " U5      :X  d   eg ! , (       d  f       N+= f)Nr   ri  r   r   rj  )	rH   r   r   r   r   r   r   r   r   )r   expected_scorer`   r_   rk  s        ra   *test_jaccard_score_zero_division_set_valuero    s     XXy),-FXXy),-F		 	 	"g'=>I]
 
#
 FMM.1111 
#	"s   'B
B(c            	         [         R                  " / SQ/ SQ/ SQ/5      n [         R                  " / SQ/ SQ/ SQ/5      n[        XS S9u  p#pE[        U/ SQS5        [        U/ SQS5        [        U/ S	QS5        [        U/ S
QS5        [	        XSS S9nUn[        U/ SQS5        [        XSS9u  p#pE[        US5        [        US5        [        US5        Ub   e[        [	        XSSS9[         R                  " U5      5        [        XSS9u  p#pE[        US5        [        US5        [        US5        Ub   e[        [	        XSSS9SU-  U-  SU-  U-   -  5        [        XSS9u  p#pE[        US5        [        US5        [        US5        Ub   e[        [	        XSSS9[         R                  " XgS95        [        XSS9u  p#pE[        US5        [        US5        [        US5        Ub   e[        [	        XSSS9S5        g )NrC   r   r   r   r   rC   r   r   r  rC   r   rC   r   r   )r   r   r   r   r<   )r   r   r   r   )r   r   rC   r   )rC   rC   rC   rC   r   r   )r   r  rC   r   r   g      ?r   g?r   r   r   r   r  r   rH   r   r!   r4   r   r3   r   r   r`   r_   rZ   r   r   r   f2rj   s           ra   +test_precision_recall_f1_score_multilabel_1rx    s   
 XX|\<@AFXX|\<@AF0NJA! a!5q9a!5q9a!7;aq1	V!T	:BGb/15 1QJA!7#3+,99FG<bggbk
 1QJA!33399FG<	!a1q519% 1TJA!7#3+,99FJ?


2' 1SJA!33399FINPSTrc   c            	         [         R                  " / SQ/ SQ/ SQ/5      n [         R                  " / SQ/ SQ/ SQ/5      n[        XS S9u  p#pE[        U/ SQS5        [        U/ S	QS5        [        U/ S
QS5        [        U/ SQS5        [	        XSS S9nUn[        U/ SQS5        [        XSS9u  p#pE[        US5        [        US5        [        US5        Ub   e[        [	        XSSS9SU-  U-  SU-  U-   -  5        [        XSS9u  p#pE[        US5        [        US5        [        US5        Ub   e[        [	        XSSS9[         R                  " U5      5        [        XSS9u  p#pE[        US5        [        US5        [        US5        Ub   e[        [	        XSSS9[         R                  " XgS95        [        XSS9u  p#pE[        US5        [        US5        [        US5        Ub   e[        [	        XSSS9SS5        g )Nrq  rr  r  r   r   r   rC   r  r   )r   r   r   r   r<   )r   r   r   r   )r   gQ?r   r   rC   r<   rC   r   rt  )r   皙?r   r   r         ?r   r   r   g      ?rH  r   r   rk   r  r   g&S?ru  rv  s           ra   +test_precision_recall_f1_score_multilabel_2r~  3  s    XX|\<@AFXX|\<@AF 1NJA!a!5q9a!5q9a!6:aq1	V!T	:BGb/150QJA!4 4 0199FG<	!a1q519%
 1QJA!4 5!6"99FG<bggbk 1TJA!5!5!=)99FJ?


2'
 1SJA! 5!5!=)99FI>rc   z%zero_division, zero_division_expected)r   r   r  c           
         [         R                  " / SQ/ SQ/ SQ/5      n[         R                  " / SQ/ SQ/ SQ/5      n[        X#S U S9u  pEpg[        XASSS/S	5        [        USS
SU/S	5        Sn[        XhSSU/S	5        [        U/ SQS	5        [	        X#S	S U S9n	Un
[        XSSU/S	5        [        X#SU S9u  pEpg[         R
                  " U5      (       a  SOUnS[         R
                  " U5      (       + -   n[        US	U-   U-  5        [        USU-   U-  5        Sn[        Xh5        Ub   e[        [	        UUS	SU S9[        U	S S95        [        X#SU S9u  pEpg[        US5        [        US
5        [        US5        Ub   e[        [	        X#S	SU S9SU-  U-  SU-  U-   -  5        [        X#SU S9u  pEpg[        XAS:X  a  SOS5        [        US
5        Sn[        USU-  5        Ub   e[        [	        X#S	SU S9[        XS95        [        X#SS9u  pEpg[        US5        [        US5        [        US5        Ub   eS n[        [	        X#S	SU S9US	5        g )!Nrr  rq  r  rc  rz  rj  r   r   r<   r   r   r   rC   r{  r   r   r   r|  r   r         ?g?r  r   rn   r   r   r   r   g@r   r   rk   gZd;O?)rH   r   r!   r4   r   r  r3   r7   )r   zero_division_expectedr`   r_   rZ   r   r   r   
expected_frw  rj   value_to_sumvalues_to_averageexpected_results                 ra   7test_precision_recall_f1_score_with_an_empty_predictionr  t  s    XX|\<@AFXX|\<@AF 1MJA! a#sC!H!La#sC1G!H!LJagq*!EqIaq1	V!T	WBGbtQ
"CQG0}JA! !7881>TL*@!AABA,0AABC,.2CCD J&99'	
 	B%	 1}JA! 5!30199G=	
 
!a1q519%	 1
-JA! a$?5SI3M->>?99Jm	
 	B(	 1SJA! 5!5!5!99OI]	
 		rc   r   )r   r   r   r   c           	      >   [         R                  " S5      n[         R                  " U5      n[        R                  " 5          [        R
                  " S5        [        UUUU US9u  pVpx[        UUU UUS9n	S S S 5        Wb   e[         R                  " U5      (       a+  WWWW	4 H   n
[         R                  " U
5      (       a  M    e   g [        U5      n[        WU5        [        WU5        [        WU5        [        W	[        U5      5        g ! , (       d  f       N= f)Nro   r   r   r   r   r   r  )rH   r   rI  r   r   r   r!   r   r  r|   r3   )r   r   r   r`   r_   rZ   r   r   r   fbetar   s              ra   "test_precision_recall_f1_no_labelsr    s     XXgF]]6"F		 	 	"g&4'

a '
 
#" 99 
xx!Q&F88F#### '-(M=)=)=)u]34= 
#	"s   4D
Dc                    [         R                  " S5      n[         R                  " U5      n[        n[        R
                  " [        5         U" XU SS9u  pEpgS S S 5        [        WS5        [        WS5        [        WS5        Wb   e[        R
                  " [        5         [        XU SS9nS S S 5        [        WS5        g ! , (       d  f       Nr= f! , (       d  f       N,= f)Nr  r   r  r   )	rH   r   rI  r!   r   r  r   r3   r   )	r   r`   r_   funcrZ   r   r   r   r  s	            ra   1test_precision_recall_f1_no_labels_check_warningsr    s    XXgF]]6"F*D	,	-&'D
a 
. 11199	,	-FG#F 
. q! 
.	- 
.	-s   C$C
C
C$c           	         [         R                  " S5      n[         R                  " U5      n[        R                  " 5          [        R
                  " S5        [        UUS SU S9u  p4pV[        XSS U S9nS S S 5        [         R                  " U 5      n [        WX U /S5        [        WX U /S5        [        WX U /S5        [        W/ SQS5        [        WX U /S5        g ! , (       d  f       Np= f)Nr  r   r   r  r  r<   ri  )
rH   r   rI  r   r   r   r!   r   r  r4   )r   r`   r_   rZ   r   r   r   r  s           ra   /test_precision_recall_f1_no_labels_average_noner    s    XXgF]]6"F 
	 	 	"g&4'

a d-
 
# JJ}-Ma-!NPQRa-!NPQRa-!NPQRaA.emM%RTUV) 
#	"s   3C
C-c                     [         R                  " S5      n [         R                  " U 5      n[        R                  " [
        5         [        XS SS9u  p#pES S S 5        [        W/ SQS5        [        W/ SQS5        [        W/ SQS5        [        W/ SQS5        [        R                  " [
        5         [        XSS S9nS S S 5        [        W/ SQS5        g ! , (       d  f       N= f! , (       d  f       N/= f)Nr  rC   r  ri  r<   rt  )	rH   r   rI  r   r  r   r!   r4   r   )r`   r_   rZ   r   r   r   r  s          ra   4test_precision_recall_f1_no_labels_average_none_warnr  7  s    XXgF]]6"F 
,	-4Dq

a 
.
 aA.aA.aA.aA.	,	-FDA 
. eY2 
.	- 
.	-s   C4C)
C&)
C7c            	      (   [         [        pS HY  nSn[        R                  " XS9   U " / SQ/ SQUS9  S S S 5        Sn[        R                  " XS9   U " / SQ/ SQUS9  S S S 5        M[     Sn[        R                  " XS9   U " [        R
                  " S	S
/S	S
//5      [        R
                  " S	S
/S
S
//5      SS9  S S S 5        Sn[        R                  " XS9   U " [        R
                  " S	S
/S
S
//5      [        R
                  " S	S
/S	S
//5      SS9  S S S 5        Sn[        R                  " XS9   U " [        R
                  " S	S	/S	S	//5      [        R
                  " S
S
/S
S
//5      SS9  S S S 5        Sn[        R                  " XS9   U " [        R
                  " S
S
/S
S
//5      [        R
                  " S	S	/S	S	//5      SS9  S S S 5        Sn[        R                  " XS9   U " S	S	/SS/SS9  S S S 5        Sn[        R                  " XS9   U " SS/S	S	/SS9  S S S 5        [        R                  " SS9 n[        R                  " S5        [        S
S
/S
S
/SS9  Sn[        UR                  5       R                  5      U:X  d   eSn[        UR                  5       R                  5      U:X  d   eSn[        UR                  5       R                  5      U:X  d   e S S S 5        g ! , (       d  f       GN= f! , (       d  f       GM  = f! , (       d  f       GN`= f! , (       d  f       GN= f! , (       d  f       GN= f! , (       d  f       GN= f! , (       d  f       GNl= f! , (       d  f       GNS= f! , (       d  f       g = f)NNr   r   zPrecision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.r   r   rC   rC   r<   r   zRecall is ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.zPrecision is ill-defined and being set to 0.0 in samples with no predicted labels. Use `zero_division` parameter to control this behavior.rC   r   r   zRecall is ill-defined and being set to 0.0 in samples with no true labels. Use `zero_division` parameter to control this behavior.Precision is ill-defined and being set to 0.0 due to no predicted samples. Use `zero_division` parameter to control this behavior.r   zRecall is ill-defined and being set to 0.0 due to no true samples. Use `zero_division` parameter to control this behavior.r   rU   Tr   r   F-score is ill-defined and being set to 0.0 due to no true nor predicted samples. Use `zero_division` parameter to control this behavior.)r!   r   r   r  rH   r   r   r   r   r   popr   )r   rK  r   r   r   s        ra   test_prf_warningsr  S  s.   *,Bq. 	 \\!'iG4 ( 	 \\!'iG4 ('! /*	  
a	#	"((QFQF#
$bhhAA/?&@)T 
$	  
a	#	"((QFQF#
$bhhAA/?&@)T 
$
	  
a	#	"((QFQF#
$bhhAA/?&@'R 
$	  
a	#	"((QFQF#
$bhhAA/?&@'R 
$
	  
a	#	1a&2r(H- 
$	  
a	#	2r(QFH- 
$ 
	 	 	-h''AAI 	
 6::<''(C/// 	 6::<''(C/// 	 6::<''(C///- 
.	-K (' (' 
$	# 
$	# 
$	# 
$	# 
$	# 
$	# 
.	-sl   K2L=L=L)7=L;=M-MM1B'N2
L	
L	
L&)
L8;
M

M
M.1
N 
Nc           	         [         R                  " 5          [         R                  " S5        S H!  n[        / SQ/ SQXS9  [        / SQ/ SQXS9  M#     [        [        R
                  " SS/SS//5      [        R
                  " SS/SS//5      SU S9  [        [        R
                  " SS/SS//5      [        R
                  " SS/SS//5      SU S9  [        [        R
                  " SS/SS//5      [        R
                  " SS/SS//5      S	U S9  [        [        R
                  " SS/SS//5      [        R
                  " SS/SS//5      S	U S9  [        SS/S
S
/SU S9  [        S
S
/SS/SU S9  S S S 5        [         R                  " SS9 n[         R                  " S5        [        SS/SS/SU S9  [        U5      S:X  d   e S S S 5        g ! , (       d  f       Nd= f! , (       d  f       g = f)Nr   r  r   r  rj  rC   r   r   r   r   rU   Tr   r   )r   r   r   r!   rH   r   ry   )r   r   r   s      ra   )test_prf_no_warnings_if_zero_division_setr    s   		 	 	"g& 3G+9g ,9g 3 	(HHq!fq!f%&HHq!fq!f%&'		
 	(HHq!fq!f%&HHq!fq!f%&'		
 	(HHq!fq!f%&HHq!fq!f%&'		
 	(HHq!fq!f%&HHq!fq!f%&'		
 	(FRHhm	
 	(Hq!fhm	
a 
#h 
	 	 	-h''FQFHM	
 6{a 
.	-i 
#	"h 
.	-s   EG8G#
G #
G1c           	         [         R                  " 5          [         R                  " S5        [        [        R
                  " SS/SS//5      [        R
                  " SS/SS//5      SU S9  S S S 5        [         R                  " SS9 n[         R                  " S5        [        [        R
                  " SS/SS//5      [        R
                  " SS/SS//5      SU S9  U S	:X  a*  [        UR                  5       R                  5      S
:X  d   eO[        U5      S:X  d   e[        SS/SS/5        U S	:X  a)  [        UR                  5       R                  5      S
:X  d   eS S S 5        g ! , (       d  f       GN= f! , (       d  f       g = f)Nr   rC   r   r   rj  Tr   r   r   r  )
r   r   r   r#   rH   r   r   r  r   ry   r   r   s     ra   test_recall_warningsr    sc   		 	 	"g&HHq!fq!f%&HHq!fq!f%&'		
 
# 
	 	 	-h'HHq!fq!f%&HHq!fq!f%&'		
 F"FJJL(() ." "" v;!###aVaV$F"FJJL(() ." ""+ 
.	- 
#	" 
.	-s   AE)	CE;)
E8;
F	c           	         [         R                  " SS9 n[         R                  " S5        [        [        R
                  " SS/SS//5      [        R
                  " SS/SS//5      SU S9  U S:X  a*  [        UR                  5       R                  5      S	:X  d   eO[        U5      S:X  d   e[        SS/SS/5        U S:X  a)  [        UR                  5       R                  5      S	:X  d   eS S S 5        [         R                  " 5          [         R                  " S
5        [        [        R
                  " SS/SS//5      [        R
                  " SS/SS//5      SU S9  S S S 5        g ! , (       d  f       N= f! , (       d  f       g = f)NTr   r   rC   r   r   rj  r   r  r   )
r   r   r   r"   rH   r   r   r  r   ry   r  s     ra   test_precision_warningsr  !	  sc   		 	 	-h'HHq!fq!f%&HHq!fq!f%&'		
 F"FJJL(() ." "" v;!###AA'F"FJJL(() ." ""+ 
.6 
	 	 	"g&HHq!fq!f%&HHq!fq!f%&'		
 
#	"7 
.	-6 
#	"s   CE)	AE:)
E7:
Fc           
         [         R                  " SS9 n[         R                  " S5        [        [	        [
        SS94 GH   nU" [        R                  " SS/SS//5      [        R                  " SS/SS//5      SU S	9  [        U5      S:X  d   eU" [        R                  " SS/SS//5      [        R                  " SS/SS//5      SU S	9  [        U5      S:X  d   eU" [        R                  " SS/SS//5      [        R                  " SS/SS//5      SU S	9  U S
:X  a,  [        UR                  5       R                  5      S:X  d   eGM  [        U5      S:X  a  GM!   e   S S S 5        g ! , (       d  f       g = f)NTr   r   r<   r   rC   r   r   rj  r   r  )r   r   r   r   r   r   rH   r   ry   r   r  r   )r   r   rk  s      ra   test_fscore_warningsr  I	  sp   		 	 	-h'! <=E1a&1a&)*1a&1a&)*+	 v;!###1a&1a&)*1a&1a&)*+	 v;!###1a&1a&)*1a&1a&)*+	 &

,,- 2- -- 6{a'''? > 
.	-	-s   EE.!E..
E<c            	      ~   / SQn / SQnSn[         R                  " / SQ/ SQ/ SQ/5      n[         R                  " / SQ/ SQ/ SQ/5      nSnXU4X4U44 HU  u  pgn[        [        [        [        [        S	S
94 H-  n	[        R                  " [        US9   U	" Xg5        S S S 5        M/     MW     g ! , (       d  f       ME  = f)N)rC   r<   r   r   rC   r<   r   rC   rP  r   r   r   r8  rO  r<   r   r   )
rH   r   r"   r#   r   r   r   r   r   r   )
	y_true_mc	y_pred_mcmsg_mc
y_true_ind
y_pred_indmsg_indr`   r_   r   r   s
             ra   'test_prf_average_binary_data_non_binaryr  p	  s    II	1 
 9i;<J9i;<J	<  
v&	) 
 Ka(	
F z5v& 65
	  65s   	B--
B<c                  .   Sn SnSnSnSnSnU [         R                  " / SQ/ SQ/ S	Q/5      4U [         R                  " S
S/SS
/SS//5      4U/ SQ4U/ SQ4U/ SQ4U[         R                  " S/S/S//5      4U[         R                  " S
/S/S//5      4U[         R                  " S/S/S//5      4U[         R                  " S
S/SS/SS//5      4U[         R                  " SS/SS/SS//5      4/
n0 X 4U _X4U_X"4U_X4S _X 4S _X!4U_X34S _XD4S _XU4S _X4S _X4S _X#4S _XC4S _XS4S _X4S _X4S _X$4S _XT4S X4S X4S X%4S 0En[        USS9 GH  u  u  pu  p XxU
4   nUc  [        R
                  " [        5         [        X5        S S S 5        X:w  a@  SR                  X5      n[        R
                  " [        US9   [        X5        S S S 5        M  XX4;  a@  SR                  U5      n[        R
                  " [        US9   [        X5        S S S 5        M  M  [        X5      u  pnnX:X  d   eUR                  S5      (       a%  UR                  S:X  d   eUR                  S:X  d   eO@[        U[         R                  " U	5      5        [        U[         R                  " U5      5        [        R
                  " [        5         [        U	S S U5        S S S 5        GM     SS/n	S S!/nS"n[        R
                  " [        US9   [        X5        S S S 5        g ! [         a
    XzU4   n GNf = f! , (       d  f       GN= f! , (       d  f       GM
  = f! , (       d  f       GM  = f! , (       d  f       GM0  = f! , (       d  f       g = f)#Nmultilabel-indicator
multiclassrU   
continuouszmulticlass-multioutputzcontinuous-multioutputr   r   r   r   rC   )r<   r   rC   )r   r  r   r<   r   r   r  r   r   r\  g?g      @)r  z@Classification metrics can't handle a mix of {0} and {1} targetsr   z{0} is not supported
multilabelcsrr   )rC   r<   )r   r<   r   )r<   )r   r<   zYou appear to be using a legacy multi-label data representation. Sequence of sequences are no longer supported; use a binary array or sparse matrix instead - the MultiLabelBinarizer transformer can convert to this format.)rH   r   r   KeyErrorr   r   r   r%   format
startswithr5   squeeze)INDMCBINCNTMMCMCNEXAMPLESEXPECTEDtype1r   type2r   r  r   merged_typey1outy2outr~   r   s                      ra   test__check_targetsr  	  s%    !C	B
C
C
"C
"C 
bhh	9i89:	bhhAAA/01	Y	i	o	RXXsQC!o&'	bhhaS1#'(	bhhuse,-.	bhhAAA/01	bhhc
S#Jc
;<=H	
C	" 

C 
	4	
 

D 
	2 

D 

D 

D 

D 
	4 

D 

D 

D  

D!" 
	4#$ 

D%& 

D	
D		4	
D-H2 %,HQ$? [e	.u-H z*r& + ~--3VE-A  ]]:W="2* >= b.4;;EBGzA&r. BA / ,:"+A(Kq***%%l33||u,,,||u,,,"5"**R.9"5"**R.9z*r#2w+ +*A %@H )	B
B	3  
z	-r 
.	-S  	.u-H	. +* >= BA +* 
.	-sN   L$L;MM M3N$L87L8;
M
	
M	 
M0	3
N	
Nc                      Sn [         R                  " [        [        R                  " U 5      S9   [        [        R                  " / 5      [        R                  " / 5      5        S S S 5        g ! , (       d  f       g = f)NzIFound empty input array (e.g., `y_true` or `y_pred`) while a minimum of 1r   )r   r   r   reescaper%   rH   r   r  s    ra   *test__check_targets_raises_on_empty_inputsr  	  sC    
UC	z3	8rxx|RXXb\2 
9	8	8s   5A..
A<c                  <    SS/n SS/n[        X5      S   S:X  d   eg )Nr   rC   r   r  )r%   re  s     ra   Atest__check_targets_multiclass_with_both_y_true_and_y_pred_binaryr  	  s,    VFWF&)!,<<<rc   zy, target_typerU   r<   r  r   r8  r9  r  c                    US;   a.  [         R                  " [        SS9   [        X 5        SSS5        g[        X 5      u  p#pEUS:X  d   eUR                  S:X  d   eUR                  S:X  d   eg! , (       d  f       g= f)z?Check correct behaviour when different target types are sparse.)rU   r  z+Sparse input is only supported when targetsr   Nr  r  )r   r   	TypeErrorr%   r  )rW   target_typey_type
y_true_out
y_pred_outr~   s         ra   !test__check_targets_sparse_inputsr  	  s     ..]]J
 1 
 
 -;1,@)J////  E)))  E)))
 
s   A00
A>c                     [         R                  " / SQ5      n [         R                  " / SQ5      n[        X5      S:X  d   e[         R                  " / SQ5      n [         R                  " / SQ5      n[        X5      S:X  d   eg )N)r   rC   rC   r   )g      !r   r  g333333ӿr   )r   r<   r<   r   )rH   r   r   r`   pred_decisions     ra   test_hinge_loss_binaryr  
  sb    XXn%FHH34Mf,777XXl#FHH34Mf,777rc   c            
         [         R                  " / SQ/ SQ/ SQ/ SQ/ SQ/ SQ/5      n [         R                  " / SQ5      n[         R                  " SU S   S   -
  U S   S   -   SU S   S   -
  U S   S	   -   SU S	   S	   -
  U S	   S
   -   SU S
   S   -
  U S
   S	   -   SU S   S
   -
  U S   S	   -   SU S   S	   -
  U S   S
   -   /5      n[         R                  " USS US9  [         R                  " U5      n[	        X5      U:X  d   eg )N
ףp=
?(\ſ(\gGz)HzGgGz׿Q޿r  333333r  RQؿr  )r  r  r  r  gzGgHzGgHzGѿgQ?)r   rC   r<   rC   r   r<   rC   r   r<   r   r   r   outrH   r   clipr   r   )r  r`   dummy_lossesdummy_hinge_losss       ra   test_hinge_loss_multiclassr   
  sH   HH((((((	
	M XX()F88a ##mA&6q&99a ##mA&6q&99a ##mA&6q&99a ##mA&6q&99a ##mA&6q&99a ##mA&6q&99	
	L GGL!T|4ww|,f,0@@@@rc   c                      [         R                  " / SQ5      n [         R                  " / SQ/ SQ/ SQ/ SQ/5      nSn[        R                  " [        US9   [        X5        S S S 5        g ! , (       d  f       g = f)N)r   rC   r<   r<   )gRQ?g rh?g(\gffffffr  r  zDPlease include all labels in y_true or pass labels as third argumentr   )rH   r   r   r   r   r   )r`   r  error_messages      ra   :test_hinge_loss_multiclass_missing_labels_with_labels_noner  ;
  s^    XXl#FHH((((		
M 	O  
z	76) 
8	7	7s   A++
A9c            
         [         R                  " / SQ5      n [         R                  " / SQ5      nSn[        R                  " [        [
        R                  " U5      S9   [        XS9  S S S 5        [         R                  " SS/SS/SS/SS/SS/SS/SS//5      n/ S	QnS
n[        R                  " [        [
        R                  " U5      S9   [        XUS9  S S S 5        g ! , (       d  f       N= f! , (       d  f       g = f)N)r<   rC   r   rC   r   rC   rC   )r   rC   r<   rC   r   r<   rC   zThe shape of pred_decision cannot be 1d arraywith a multiclass target. pred_decision shape must be (n_samples, n_classes), that is (7, 3). Got: (7,)r   r  r   rC   r<   r   zThe shape of pred_decision is not consistent with the number of classes. With a multiclass target, pred_decision shape must be (n_samples, n_classes), that is (7, 3). Got: (7, 2))r`   r  rv   )rH   r   r   r   r   r  r  r   )r`   r  r  rv   s       ra   <test_hinge_loss_multiclass_no_consistent_pred_decision_shaper  L
  s     XX+,FHH23M	  
z=)A	B&> 
C HHq!fq!fq!fq!fq!fq!fqRSfUVMF	  
z=)A	B&fM 
C	B 
C	B 
C	Bs    
C$C5$
C25
Dc            	         [         R                  " / SQ/ SQ/ SQ/ SQ/ SQ/5      n [         R                  " / SQ5      n[         R                  " / SQ5      n[         R                  " SU S   S   -
  U S   S   -   SU S   S   -
  U S   S   -   SU S   S   -
  U S   S	   -   SU S	   S   -
  U S	   S   -   SU S
   S   -
  U S
   S	   -   /5      n[         R                  " USS US9  [         R                  " U5      n[	        XUS9U:X  d   eg )Nr  皙r  r  r  r  )r   rC   r<   rC   r<   r   rC   r<   r   rC   r   r<   r   r   r  r   r  r  r`   rv   r  r  s        ra   .test_hinge_loss_multiclass_with_missing_labelsr  h
  s3   HH(((((	
M XXo&FXXl#F88a ##mA&6q&99a ##mA&6q&99a ##mA&6q&99a ##mA&6q&99a ##mA&6q&99	
L GGL!T|4ww|,fF;?OOOOrc   c            	         [         R                  " / SQ/ SQ/ SQ/ SQ/ SQ/5      n [         R                  " / SQ5      n[         R                  " / SQ5      n[         R                  " SU S   S   -
  U S   S   -   SU S   S	   -
  U S   S   -   SU S	   S	   -
  U S	   S   -   SU S
   S   -
  U S
   S	   -   SU S   S	   -
  U S   S   -   /5      n[         R                  " USS US9  [         R                  " U5      n[	        [        XUS9U5        g )N)r  r  r  )g333333ÿr  r  )r  r  r  )r  g(\gzGڿ)r   r<   r<   r   r<   r   rC   r   r<   r   r   r  r   )rH   r   r  r   r3   r   r  s        ra   @test_hinge_loss_multiclass_missing_labels_only_two_unq_in_y_truer  
  s4   
 HH!!!!!	
M XXo&FXXi F88a ##mA&6q&99a ##mA&6q&99a ##mA&6q&99a ##mA&6q&99a ##mA&6q&99	
L GGL!T|4ww|,68:Jrc   c            
         / SQn / SQ/ SQ/ SQ/ SQ/ SQ/ SQ/n[         R                  " SUS   S   -
  US   S   -   SUS   S   -
  US   S   -   SUS   S   -
  US   S	   -   SUS	   S   -
  US	   S   -   SUS
   S	   -
  US
   S   -   SUS   S   -
  US   S	   -   /5      n[         R                  " USS US9  [         R                  " U5      n[	        X5      U:X  d   eg )N)r0  r1  r2  r1  whiter2  r  r  r  r  rC   r   r<   r   r   r   r  r  )r`   r  r  r  s       ra   +test_hinge_loss_multiclass_invariance_listsr  
  s4    ?F$$$$$$M 88a ##mA&6q&99a ##mA&6q&99a ##mA&6q&99a ##mA&6q&99a ##mA&6q&99a ##mA&6q&99	
	L GGL!T|4ww|,f,0@@@@rc   c            	      &   / SQn [         R                  " SS/SS/SS/SS/SS/S	S
//5      n[        X5      n[         R                  " [        R
                  " [         R                  " U 5      S:H  US S 2S4   5      5      * n[        X#5        / SQn / SQ/ SQ/ SQ/n[        XSS9n[        US5        U S-  n US-  n[        XSS9n[        US5        / SQn SS/SS/SS//n[        R                  " [        5         [        X5        S S S 5        / SQn / SQ/ SQ/ SQ/n/ SQnS n[        R                  " [        [        R                  " U5      S!9   [        XUS"9  S S S 5        / S#Qn SS/SS/SS/SS//n[        X5      n[        US$5        SS/n S%S&/SS//n[         R                  " SS/SS//5      nS'n[        R                  " [        [        R                  " U5      S!9   [        X5        S S S 5        S%S&/SS/SS//nS(n[        R                  " [        [        R                  " U5      S!9   [        X5        S S S 5        [         R                  " [         R                  " US S 2S4   5      5      * n[        XSS/S"9n[        X5        / S)Qn / S*Q/ SQ/ S+Q/n	[        X	/ S,QS"9n[        U[         R                  " S5      * 5        g ! , (       d  f       GN= f! , (       d  f       GN= f! , (       d  f       GN= f! , (       d  f       N= f)-Nnor  r  yesr  r  r   r   rm   {Gz?Gz?r   r}  gMbP?g+?r  rC   rK  r   r   r   r   r   r   r   r   r   TrR  g躕ʀ?r<   Fg.L`@r   r   r   r   r   )rm   r   r   )r   rm   r   r   r   r   )r   r   r   zPy_true contains values {'b'} not belonging to the passed labels ['a', 'c', 'd'].r   r   hamspamr  r  CT?r   r   zy_true contains only one label (2). Please provide the list of all expected class labels explicitly through the labels argument.zBFound input variables with inconsistent numbers of samples: [3, 2]r   )r   r   r   r   r   r   rJ  )rH   r   r   r   r
   logpmfr2   r   r   r   r  r  log)
r`   r_   loss	loss_truerv   	error_strr   true_log_losscalculated_log_lossy_score2s
             ra   test_log_lossr  
  s   4FXX
sc3Z$SzD$<%QVXF F#D))"((6*:e*CVAqD\RSSID$ F@FFd3DD)$ aKF
aKFFe4DD-( FCj3*sCj1F	z	"  
# F@FF	"  
z9)=	>/ 
? ,FCj3*sCj3*=FF#DD)$ VFCj3*%Fhhc
S#J/0G	H  
z9)=	>  
? Cj3*sCj1FTI	z9)=	>  
?
 WWRVVGAqDM233M"6Aq6B'7 F/BHFY7DD266#;,'_ 
#	" 
?	>$ 
?	>
 
?	>s0   ;KK$K02L
K
K-0
K?
Lc                     [         R                  " SS/U S9n[         R                  " SS/U S9n[        X5      n[         R                  " U5      (       d   eg)zCheck the behaviour internal eps that changes depending on the input dtype.

Non-regression test for:
https://github.com/scikit-learn/scikit-learn/issues/24315
r   rC   r  N)rH   r   r   isfinite)rU  r`   r_   r
  s       ra   test_log_loss_epsr  
  sK     XXq!fE*FXXq!fE*FF#D;;trc   c                     [         R                  " / SQ5      n[         R                  " SS/SS/SS/SS//U S9n[        R                  " [        S	S
9   [        X5        SSS5        g! , (       d  f       g= f)zGCheck that log_loss raises a warning when y_pred values don't sum to 1.r  r   r   r   r   r   r   r  z$The y_prob values do not sum to one.r   N)rH   r   r   r  r  r   )rU  r`   r_   s      ra   'test_log_loss_not_probabilities_warningr    s_     XXl#FXXSzC:SzC:FeTF	k)O	P  
Q	P	Ps   A((
A6r   r   c                 N    [        X5      [        R                  " S5      :X  d   eg)z6Check that log_loss returns 0 for perfect predictions.r   N)r   r   r   re  s     ra   !test_log_loss_perfect_predictionsr  "  s      F#v}}Q'7777rc   c                  J   [         R                  " / SQ5      n [         R                  " SS/SS/SS/SS//5      n[        [        4/n SSKJnJn  UR                  XC45        U H+  u  pVU" U 5      U" U5      p[        Xx5      n	[        U	S5        M-     g ! [         a     N>f = f)	Nr  r   r   r   r   r   )	DataFramer  r  )
rH   r   r0   r  r  r  r  ImportErrorr   r2   )
y_try_prtypesr  r  TrueInputTypePredInputTyper`   r_   r
  s
             ra   test_log_loss_pandas_inputr   0  s    8823D88c3Z#sc3Z#sDED]+,E,f() ).$&t,mD.A'i(	 ).  s   	B 
B"!B"c                      [         R                  " S5      n [        R                  " [        U S9   [        / SQ/ SQ/ SQ/ SQ// SQS9  S S S 5        g ! , (       d  f       g = f	NzLabels passed were ['spam', 'eggs', 'ham']. But this function assumes labels are ordered lexicographically. Pass the ordered labels=['eggs', 'ham', 'spam'] and ensure that the columns of y_prob correspond to this ordering.r   eggsr  r  r   r8  r   )r  r$  r  r   )r  r  r   r  r  r   expected_messages    ra   test_log_loss_warningsr'  B  sK    yy	= 
k)9	:#	9-*	
 
;	:	:   A
A c                     [         R                  " / SQ5      n [         R                  " / SQ5      n[        R                  " X-
  5      S-  [	        U 5      -  n[        [        X 5      S5        [        [        X5      U5        [        [        SU -   U5      U5        [        [        SU -  S-
  U5      U5        [         R                  " SU-
  U45      n[         R                  " SU -
  U 45      n[        [        X5      U5        [        [        XC5      U5        [        [        XSS9U5        [        [        XS	S9U5        [        [        XS
S9SU-  5        [        [        S/S/5      S5        [        [        S/S/5      S5        [        [        S/S/5      S5        [        [        S/S/SS9S5        [        [        S/S/SS9S5        g )Nr   rC   rC   r   rC   rC   r   r   rm   r   r   gffffff?r<   r   r   rC   auto)scale_by_halfTFr   r   g|Gz?r   r  foobarr   )rH   r   r   normry   r3   r   column_stack)r`   y_prob
true_scorey_prob_reshapedy_true_reshapeds        ra   test_brier_score_loss_binaryr6  Q  s   XX()FXX56FV_-2S[@J(8#>(8*E(vv>
K(Va@*M ooq6z6&:;Oooq6z6&:;O(A:N(JJW v>
 t<j u=q:~
 ("u5v>(!se4f=(!se4nE(%3%5I6R%3%59rc   c            	         [        [        / SQ/ SQ/ SQ/ SQ// SQS9S5        [        [        / SQ/ SQ/ S	Q/ S
Q/5      S5        [        [        / SQ/ SQ/ SQ/ SQ/5      S5        [        [        / SQ/ SQ/ SQ/ SQ/5      S5        g )Nr#  rq  rr  )r$  r  r  yamsr   r   rK  r  r   r  gt?r   )r   r   r   )r   r   r   )r   r   r   r   r<   )r3   r   r   rc   ra    test_brier_score_loss_multiclassr9  x  s    #<62	

 	 /J	
 		 /J	
 	
	 /J	
 	
	rc   c                  z   [         R                  " / SQ5      n [         R                  " / SQ5      n[        R                  " [        5         [        XSS  5        S S S 5        [        R                  " [        5         [        XS-   5        S S S 5        [        R                  " [        5         [        XS-
  5        S S S 5        [         R                  " / SQ5      n [         R                  " / SQ/ SQ/ SQ/5      n[        R                  " [        5         [        XSS  5        S S S 5        [        R                  " [        5         [        XS-   5        S S S 5        [        R                  " [        5         [        XS-
  5        S S S 5        [         R                  " / S	Q5      n [         R                  " / S
Q5      n[        R                  " S5      n[        R                  " [        US9   [        X5        S S S 5        / SQn SS/SS/SS//nSn[        R                  " [        [        R                  " U5      S9   [        X5        S S S 5        / SQn / SQ/ SQ/ SQ/n/ SQnSn[        R                  " [        [        R                  " U5      S9   [        XUS9  S S S 5        S/n SS//nSn[        R                  " [        [        R                  " U5      S9   [        X5        S S S 5        [        [        XSS/S9S5        g ! , (       d  f       GN= f! , (       d  f       GN= f! , (       d  f       GNy= f! , (       d  f       GN#= f! , (       d  f       GN= f! , (       d  f       GN= f! , (       d  f       GN= f! , (       d  f       GND= f! , (       d  f       GN= f! , (       d  f       N= f)Nr*  r+  rC   r   rK  r  r   r  )r   rC   r<   r   r   r   r   r   zpThe type of the target inferred from y_true is multiclass but should be binary according to the shape of y_prob.r   r   r   zy_true and y_prob contain different number of classes: 3 vs 2. Please provide the true labels explicitly through the labels argument. Classes found in y_true: [0 1 2]r#  r   r8  )r$  r  r  r8  zwThe number of classes in labels is different from that in y_prob. Classes found in labels: ['eggs' 'ham' 'spam' 'yams']r   r$  rm   r   zy_true contains only one label (eggs). Please provide the list of all expected class labels explicitly through the labels argument.r  r  )	rH   r   r   r   r   r   r  r  r3   )r`   r2  r  rv   s       ra   $test_brier_score_loss_invalid_inputsr<    s   XX()FXX56F	z	", 
# 
z	"#. 
# 
z	"#. 
#
 XXi FXXIJF	z	", 
# 
z	"#. 
# 
z	"#. 
#
 XXl#FXX*+FII	AM 
z	7( 
8 F!fq!fq!f%F	  
z=)A	B( 
C %FI.F,F	/ 
 
z=)A	B7 
C XFCj\F	 
 
z=)A	B( 
C (PRVWK 
#	" 
#	" 
#	" 
#	" 
#	" 
#	" 
8	7 
C	B 
C	B 
C	Bsx   L
<L-L.M M7M$-M6?NNN,

L
L+.
L= 
M
M!$
M36
N
N
N),
N:c                      [         R                  " S5      n [        R                  " [        U S9   [        / SQ/ SQ/ SQ/ SQ// SQS9  S S S 5        g ! , (       d  f       g = fr"  )r  r  r   r  r  r   r%  s    ra   test_brier_score_loss_warningsr>    sQ    yy	= 
k)9	:#
 +	
 
;	:	:r(  c                      Sn [         R                  " [        U S9   [        / SQ/ SQ5        S S S 5        g ! , (       d  f       g = f)Nz%y_pred contains classes not in y_truer   ri  r   )r   r  r  r   r  s    ra   #test_balanced_accuracy_score_unseenr@    s+    
1C	k	-	95 
.	-	-r  zy_true,y_pred)r   r   r   r   )r   r   r   r   )r   r   r   r   c                 R   [        XS[        R                  " U 5      S9n[        5          [	        X5      nS S S 5        W[
        R                  " U5      :X  d   e[	        XSS9n[	        U [        R                  " X S   5      5      nXCU-
  SU-
  -  :X  d   eg ! , (       d  f       Nh= f)Nr   r   T)adjustedr   rC   )r#   rH   uniquer6   r   r   r   	full_like)r`   r_   macro_recallbalancedrB  chances         ra   test_balanced_accuracy_scorerH     s      		&0AL 
	*6: 
 v}}\2222&vEH$VR\\&)-LMF6)a&j9999 
	s   B
B&r   ))FTr  )r   r   )zeroonec                 0   [         R                  R                  S5      nSUS   pCUR                  XSS9nU [        L a  UR                  US9nOUR                  5       nU " XVUS9n[         R                  " [         R                  " U5      5      (       a   eg)	zCheck that the metric works with different types of `pos_label`.

We can expect `pos_label` to be a bool, an integer, a float, a string.
No error should be raised for those types.
*   rq  r   T)r  replacer  r   N)	rH   rL   rM   choicer   uniformr  anyr  )r   r   r[   rX   r   r`   r_   r  s           ra   *test_classification_metric_pos_label_typesrQ    s    * ))


#Cwr{yZZZ>F!!),Fi8Fvvbhhv&'''''rc   zy_true, y_pred, expected_scorec                 L    [        XSS9[        R                  " U5      :X  d   eg)zCheck the behaviour of `zero_division` for f1-score.

Non-regression test for:
https://github.com/scikit-learn/scikit-learn/issues/26965
r   r  N)r   r   r   )r`   r_   rn  s      ra   2test_f1_for_small_binary_inputs_with_zero_divisionrS  6  s"     F#6&--:WWWWrc   scoringr  )r   r   c           	      z    [         R                  " SS9u  p[        SSS9R                  X5      n[	        X1X SSS9  g)	aB  Check that we validate `np.nan` properly for classification metrics.

With `n_jobs=2` in cross-validation, the `np.nan` used for the singleton will be
different in the sub-process and we should not use the `is` operator but
`math.isnan`.

Non-regression test for:
https://github.com/scikit-learn/scikit-learn/issues/27563
r   )rB   r   )	max_depthrB   r<   r  )rT  n_jobserror_scoreN)r   make_classificationr+   rQ   r(   )rT  rV   rW   
classifiers       ra   9test_classification_metric_division_by_zero_nan_validatonr[  J  s;    ( ''Q7DA'!!DHHNJJ1aWUrc   c            	         / SQn / SQn[         R                  " SS/SS/SS/SS/SS	/S
S//5      n[         R                  " SS/SS/SS/SS/SS/SS//5      n[        XS9n[        XSS9n[        XSS9nSXV-  -
  nU[        R
                  " U5      :X  d   e[         R                  " / SQ5      nUS S R                  5       UR                  5       -  US S 2S4'   USS  R                  5       UR                  5       -  US S 2S4'   [        XUS9n[        U UUSS9n[        U UUSS9nSXV-  -
  nU[        R
                  " U5      :X  d   e[         R                  " SS/SS/SS/SS/SS/SS//5      n[        X5      nSUs=:  a  S:  d   e   e[        X5      n	U	[        R
                  " U5      :X  d   e[         R                  " SS/SS/SS/SS/SS/SS//5      n[        X5      nUS:  d   e[        X5      n	U	[        R
                  " U5      :X  d   e/ SQn [         R                  " SS/SS/SS/SS/SS/SS//5      n[        X5      nUS:X  d   e[        X5      n	U	S:X  d   e/ SQn / SQn[         R                  " SS/SS/SS/SS//5      n[        X5      nUS:X  d   e[        X5      n	U	S:X  d   e/ SQn[        XUS9n
U
S:X  d   e/ SQn / SQn[         R                  " / S Q/ S Q/ S!Q/ S"Q/5      n[        X5      nSUs=:  a  S:  d   e   e[        XUS9nSUs=:  a  S:  d   e   e[         R                  " / S#Q/ S$Q/ S"Q/ S%Q/5      n[        X5      nUS:  d   e[        XUS9nUS:  d   eg )&Nrf  r  r   rm   r   r   r   gffffff?g?r  r  re  F)r`   r_   rS  rC   )r<   rC   r   r   r   rC   r   r   )r`   r_   r<  )r`   r_   r<  rS  r   r   r   r   r}  )r   rC   rC   rC   )r  r  r  r  )r<   r<   r<   r<   rI  )highr]  lowneutral)ffffff?r   r   r   )r   r   r   r   r  )r   r   r   r  r  )rH   r   r'   r   r   r   r]  )r`   y_true_stringr_   y_pred_nulld2_scorelog_likelihoodlog_likelihood_nulld2_score_truer<  d2_score_stringd2_score_with_sample_weights              ra   test_d2_log_loss_scoreri  c  s\   F;MXX#J#J#J#J4L4L	
	F ((#J#J#J#J#J#J	
	K !>HVeLN"&PUV<<Mv}}]3333 HH/0M%bq)--/-2C2C2EEK1%ab)--/-2C2C2EEK1 MH #	N ##	 <<Mv}}]3333 XX#J#J#J#J#J#J	
	F !0HC'>OfmmH5555 XX#J#J#J#J4L#J	
	F !0Ha<<'>OfmmH5555  FXX#J#J#J#J#J#J	
	F !0Hq=='>Oa F/MXXd|dD\D$<$NOF 0Hq=='>Oa M"3m# '!+++ 0F(MXX		
F !0HC }MHCXX		
F !0Ha<< }MHa<<rc   c                      / SQn / SQn/ SQn[         R                  " / SQS5      n[        XX!S9n[         R                  " / SQS5      n[        XX!S9nSXF-  -
  n[        XX!S9n[	        X5        g	)
zCheck that d2_log_loss_score works when not all labels are present in y_true

non-regression test for https://github.com/scikit-learn/scikit-learn/issues/30713
r<   r   r<   r   r   )r`  r   r   r   r   r   rC   )r<  rv   )r   r   r   rC   N)rH   tiler   r'   r2   )	r`   rv   r<  r_   log_loss_obsrb  log_loss_nullexpected_d2_scorerc  s	            ra   %test_d2_log_loss_score_missing_labelsrq    sy    
 FF(MWWY'FF-WL ''-0K=M L88 mH H0rc   c                      / SQn [         R                  " / SQS5      n[        X/ SQS9n[        X/ SQS9n[        X#5        g)zGCheck that d2_log_loss_score doesn't depend on the order of the labels.rk  r   rl  r   r   r0  N)rH   rm  r'   r2   )r`   r_   rc  d2_score_others       ra   "test_d2_log_loss_score_label_orderrt    s7    FWWY'F 	BH&viHNH-rc   c                     / SQn SS/SS/SS//nSn[         R                  " [        US9   [        X5        S	S	S	5        / SQn SS/SS/SS//n/ SQnS
n[         R                  " [        US9   [        XUS9  S	S	S	5        / SQn / SQ/ SQ/nSn[         R                  " [        US9   [        X5        S	S	S	5        S/n SS//nSn[         R                  " [
        US9   [        X5        S	S	S	5        / SQn SS/SS/SS//nSn[         R                  " [        US9   [        X5        S	S	S	5        / SQn S/nSS/SS/SS//nSn[         R                  " [        US9   [        XUS9  S	S	S	5        g	! , (       d  f       GNA= f! , (       d  f       GN= f! , (       d  f       N= f! , (       d  f       N= f! , (       d  f       N= f! , (       d  f       g	= f)zLTest that d2_log_loss_score raises the appropriate errors on
invalid inputs.r   r   r   r   r   r   z#contain different number of classesr   Nz(number of classes in labels is differentr   )r   r   r   )r   r   r   rH  rC   zscore is not well-definedr   y_true contains only one labelz.The labels array needs to contain at least two)r   r   r   r'   r  r   )r`   r_   errrv   s       ra   test_d2_log_loss_score_raisesrx  !  s    FCj3*sCj1F
/C	z	-&) 
.
 FCj3*sCj1FF
4C	z	-&8 
. F/F
+C	z	-&) 
. SFCj\F
%C	,C	8&) 
9 FCj3*sCj1F
*C	z	-&) 
.
 FSFCj3*sCj1F
:C	z	-&8 
.	-O 
.	- 
.	- 
.	- 
9	8 
.	- 
.	-sG   E1-F'FF&F7G1
F 
F
F#&
F47
G
Gc                  v   / SQn / SQn/ SQn/ SQn/ SQn[        XS9n[        XS9n[        XS9nSXg-  -
  n[        R                  " U5      U:X  d   e/ SQn[        XS9nUS:X  d   e[        X#S	S
9nUS:X  d   e/ SQn[        XU S9nUS:X  d   e[        UUU S	S9nUS:X  d   e/ SQn / SQn/ SQn/ SQ/ SQ/ SQ/ SQ/ SQ/ SQ/ SQ/ SQ/ SQ/ SQ/
n[        XU S9nUS:X  d   e[        UUU S9nUS:X  d   e/ SQ/ SQ/ SQ/ SQ/ SQ/ SQ/ SQ/ SQ/ SQ/ SQ/
n[        XU S9nUS:  d   e[        UUU S9nUS:  d   eg)z]Test that d2_brier_score gives expected outcomes in both the binary and
multiclass settings.
)r<   r<   r   rC   rC   rC   )r   rC   rC   r   r   rC   )r  r  r  r  r  r  )r   r   r   r   rm   r   )r   r   r   r   r   r   )r`   y_probarC   r   r  )r`   rz  r   )r   r   r   r   r   r   )r`   rz  r<  )r`   rz  r<  r   )
r<   rC   r   rC   rC   r<   rC   r   rC   r   )
r   r   r<   r<   r<   rC   rC   rC   rC   r   )
ddr{  ccr|  r|  bbr}  r}  r}  aa)r   r   r}  r  )r   r   r   r   )r   r   r   r   )r   r   r   r   r   r   r   r   r   N)r&   r   r   r   )	r<  r`   ra  rz  y_proba_refrc  brier_score_modelbrier_score_refd2_score_expecteds	            ra   test_d2_brier_scorer  R  s   
 'MF;M -G0KV=H(H&fJO-??=="&7777 -GV=Hq==]uUHq==
 -GmH q==#	H q== 3M+FPM 	G mH q==#H
 q==
 	G mH c>>#H
 c>>rc   c                     / SQn / SQn/ SQ/ SQ/ SQ/ SQ/n[        XUS9nUS:X  d   e/ SQn[        XUS9nU[        R                  " U5      :X  d   e/ SQ/ SQ/ SQ/ SQ/n[        XUS9n[        R                  " U5      S	:X  d   eg
)zGTest that d2_brier_score gives expected outcomes when labels are passed)r   r<   r   r<   r   )r   r   r   )r`   rz  rv   r   r   r   r   N)r&   r   r   )r`   rv   rz  rc  new_d2_scoreneg_d2_scores         ra   test_d2_brier_score_with_labelsr    s    
 FF	G VVLHq== F!PL6==2222 		G "PL==&",,,rc   z!y_true, y_pred, labels, error_msg)rC   r<   rC   r   r;  z7inferred from y_true is multiclass but should be binaryr  r  r  r  zpos_label is not specified)r   rC   r   r   rC   rC   r   z.variables with inconsistent numbers of samples)r   rC   r   rC   )g?r   r   r   z%y_prob contains values greater than 1)gr   r   r   z"y_prob contains values less than 0r   rv  rs  )r<   r   r   r<   )r   r   r   r   )r   r   r   r   z(Multioutput target data is not supportedr   )r   r   r   z"not belonging to the passed labelsri  z*labels array needs to contain at least twoc                     [         R                  " U 5      n [         R                  " U5      n[        R                  " [        US9   [        XUS9  SSS5        g! , (       d  f       g= f)zITest that d2_brier_score raises the appropriate errors
on invalid inputs.r   r   N)rH   asarrayr   r   r   r&   )r`   r_   rv   	error_msgs       ra   test_d2_brier_score_raisesr    sE    | ZZFZZF	z	3vf5 
4	3	3s   A
A(c                      [         R                  " S/5      n [         R                  " S/5      nSn[        R                  " [        US9   [        X5        SSS5        g! , (       d  f       g= f)zMTest that d2_brier_score emits a warning when there are less than
two samplesrC   r   z+not well-defined with less than two samplesr   N)rH   r   r   r  r   r&   )r`   r_   warning_messages      ra   4test_d2_brier_score_warning_on_less_than_two_samplesr    sK     XXqc]FXXse_FCO	,O	Dv& 
E	D	Ds   
A
A-zarray_namespace, device, _c                 V   [        X5      nUR                  / SQUS9nUR                  / SQUS9nUR                  / SQUS9n[        SS9   [        XEUS9n[	        U5      S   [	        U5      S   :X  d   e[        U5      [        U5      :X  d   e SSS5        g! , (       d  f       g= f)	zTest that `confusion_matrix` works for all array types when `labels` are passed
such that the inner boolean `need_index_conversion` evaluates to `True`.rJ  r,   )r   r   r   Tarray_api_dispatchr   r   N)r1   r  r   r   r.   array_api_device)array_namespacer-   r~   xpr`   r_   rv   r  s           ra   test_confusion_matrix_array_apir  #  s     
o	6BZZ	&Z1FZZ	&Z1FZZ	&Z1F	4	0!&@V$Q'=+@+CCCC'+;F+CCCC 
1	0	0s   AB
B(prob_metric
str_y_trueuse_sample_weightz$array_namespace, device_, dtype_namec                    [        X45      nU(       a  [        R                  " / SQ5      OSn0 nU(       aD  [        R                  " / SQ5      n	[        R                  " U	5      n
SU R                  ;   a  SUS'   O'[        R                  " / SQ5      n	UR                  XS9n
[        R                  " / S	QUS
9nUR                  XS9nU " X4SU0UD6n[        SS9   U " X4SU0UD6nSSS5        W[        R                  " U5      :X  d   eU(       a/  [        R                  " / SQ5      n	[        R                  " U	5      n
O'[        R                  " / SQ5      n	UR                  XS9n
[        R                  " / SQ/ SQ/ SQ/ SQ/US
9nUR                  XS9nU " X5      n[        SS9   U " X5      nSSS5        U[        R                  " U5      :X  d   eg! , (       d  f       N= f! , (       d  f       N<= f)zTest that :func:`brier_score_loss`, :func:`log_loss`, func:`d2_brier_score`
and :func:`d2_log_loss_score` work correctly with the array API for binary
and mutli-class inputs.
r  Nr  brierr  r   rs  r,   )r   r   r   r   r  r<  Tr  )r   r   r   r   r  r  )r   r   r   r   )r   r   r   r   )r   r   r   r   )r1   rH   r   r  __name__r   r   r   )r  r  r  r  device_
dtype_namer  r<  extra_kwargs	y_true_npy_true_xp_or_np	y_prob_np	y_prob_xpmetric_score_npmetric_score_xps                  ra   $test_probabilistic_metrics_array_apir  5  s    
o	7B.?BHH\*TM LHH78	**Y/k*** ).L%HH\*	**Y*?-Z@I

9
5I!,9=IO 
4	0%
6C
GS
 
1
 fmmO<<<< HH12	**Y/HH\*	**Y*?    		
 I 

9
5I!)7O	4	0%oA 
1 fmmO<<<<; 
1	04 
1	0s   G)	G(
G%(
G6c                    [        X#5      nU(       a  [        R                  " / SQ5      OSn[        R                  " / SQ/ SQ/ SQ/ SQ/US9nUR                  XsS9n[        R                  " / S	Q/ S
Q/ SQ/ SQ/US9n	UR                  XS9n
U " XyUS9n[	        SS9   U " XUS9nSSS5        W[
        R                  " U5      :X  d   eg! , (       d  f       N+= f)zTest that :func:`brier_score_loss`, :func:`log_loss`, func:`d2_brier_score`
and :func:`d2_log_loss_score` work correctly with the array API for
multi-label inputs.
r  Nr  rs  rr  )rC   rC   r   rC   r  r,   )r  gHzG?gq=
ףp?gQ?)r  gRQ?Q?gq=
ףp?)r  gQ?gQ?g)\(?)gQ?gףp=
?gp=
ף?g(\?rI  Tr  )r1   rH   r   r  r   r   r   )r  r  r  r  r  r  r<  r  	y_true_xpr  r  r  r  s                ra   /test_probabilistic_metrics_multilabel_array_apir  y  s     
o	7B.?BHH\*TM		
 I 

9
5I$$$$		
 I 

9
5I!)mTO	4	0%i-X 
1 fmmO<<<< 
1	0s   C


C)NF)r  r   	functoolsr   	itertoolsr   r   r   numpyrH   r   scipyr   r   scipy.spatial.distancer	   rJ  scipy.statsr
   sklearnr   r   sklearn.baser   sklearn.datasetsr   sklearn.exceptionsr   sklearn.metricsr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r    r!   r"   r#   r$   sklearn.metrics._classificationr%   r&   r'   sklearn.model_selectionr(   sklearn.preprocessingr)   r*   sklearn.treer+   sklearn.utils._array_apir-   r  r.   r/   sklearn.utils._mockingr0   sklearn.utils._testingr1   r2   r3   r4   r5   r6   sklearn.utils.extmathr7   sklearn.utils.fixesr8   r9   sklearn.utils.validationr:   rb   r   markparametrizert  r   r   r   r   r   r   r   r   r   r   r   r   r  r  r$  r4  rF  rL  rZ  ra  rd  rk  ro  rw  r{  r  r  r   r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r   r  r  r"  r'  r)  r+  r.  r5  r8  r;  r@  rB  rD  rF  rL  rT  rW  re  rg  rl  ro  rx  r~  r  r  r  r  r  r  r  r  r  r  r  r  r  r  
csr_matrixr  r  r  r  r  r  r  r  r  r  r  float16r  r  r  r   r'  r6  r9  r<  r>  r@  rH  rQ  rS  thread_unsafer[  ri  rq  rt  rx  r  r  r  r  r  r  r  r   rc   ra   <module>r     s   	   2 2     8 ! ! ' ; 5     . 
 4 @ / 1  . > 7)(`>DB 61a*@A B& T{aVUOi=OP))(7D PQ R$ PQ*J R*JZ PQM RM,>6  HH####	
	
 .=	
(9)(9  HH#####	
 	
*;+*;
 =*	= OF .9.9%S : :%SPT< +%%;&) 
 ((#56((#56 N	
 ((#56((#56
B		
 ((#56((#56 N	
 ((#56((#56 H	
;$'P*Q'P*  ((?3((?3
<		
**"4 4-8M 9M;0 		S!E"C C 





 $S	!3'	$bff-S	!3'	'0		D	D$ $S	!3'	$c*	$c*	'0		D	D$>1 1a.9)aS1#J<8!$	' 9 :' )aS1#J<8!$	 9$2N RF5p c5\2U 3UB7(t $ST, U,0"(".& 	:;
QAB 
'(  886 "AB0 C0"%4%$%(%6 %F%,%.XI PQ% R%<6B(%DPJZ"J.+ 868:LM	2 N	2 PQ?U R?UD PQ= R=@ PQ+&&266266"23[	 R
[| !%$MN1a.9"5 : O &"5J $MN" O"& 1a.9 W : WF38e0P 1a.9:  :: z 61a*@A$ B$N 61a*@A$
 B$
N 61a*@A#( B#(L'>_D3= 			QC!qcA3/	0(;			QC!qcA3/	0,?			Iy)<	=|L** 8A6*"N8P4@A8I(X 2::rzz2::"FG
 H
 2::rzz2::"FG! H! 	I	aVaVaV,-	Y	95688)$
$N FIXX
&6 	34	34	34
:
: #&' C(($ $	1a&	288QF+S1	1a&	288QF+S1	1a&	288QF+S1	1a&	288QF+S1	XX HBFF3Karvv>O266:L7	V VSl1:..9b`F-@ '  E		
 ' (		
 " <		
  3		
 !0		
 3Z#sc3Z0,		
 <(!#786		
 o?F0		
 o?C8		
c7:v6w:v6'  "K"MDD $h@QR t}5,udm<*,U,W9= = 69=x $h@QR ,udm<*,U,W!= =!=rc   