
    9i1                     (   S SK r S SKrS SKrS SKrS SKrS SKrS SKrS SKrS SKrS SK	r	S SK
Jr  S SKJr  S SKJr  S SKrS SKrS SKJr  S SKJrJr  SqSr\R4                  R7                  S	S5      r\R4                  R7                  S
S5      rSr\R4                  R7                  SS5      r\ " \S5      (       d  \RB                  \l"        S r#S r$S r%S r&S\'4S jr( " S S5      r)S r*S\+4S jr,S r-   SS jr.Sr/ " S S\	R`                  5      r1g)     N)OrderedDict)Mapping)
expanduser)DEFAULT_CREDENTIALS_PATH)is_tf_availableis_torch_available   
TEST_LEVELTEST_ACCESS_TOKEN_CITESTTEST_ACCESS_TOKEN_SDKDEVu   内部测试模型TEST_MODEL_ORGcitestNaNc                  L    [        [        5      n [        R                  " U SS9  g )NT)ignore_errors)r   r   shutilrmtree)path_credentials    [/var/www/html/land-doc-ocr/venv/lib/python3.13/site-packages/modelscope/utils/test_utils.pydelete_credentialr   &   s     !9:O
MM/6    c                  ~    [         [        R                  ;   a   [        [        R                  [            5      q[        $ N)TEST_LEVEL_STRosenvironintr
    r   r   
test_levelr   +   s'    #N34
r   c                 \    [        5       (       d  [        R                  " S5      " U 5      n U $ )Nztest requires TensorFlow)r   unittestskip	test_cases    r   
require_tfr%   3   s%    MM"<=iH	r   c                 \    [        5       (       d  [        R                  " S5      " U 5      n U $ )Nztest requires PyTorch)r   r!   r"   r#   s    r   require_torchr'   9   s%    MM"9:9E	r   levelc                     U q g r   )r
   )r(   s    r   set_test_levelr*   ?   s    Jr   c                   *    \ rS rSrSS jrS rS rSrg)DummyTorchDatasetD   Nc                 (    Xl         X l        X0l        g r   featlabelnum)selfr0   r1   r2   s       r   __init__DummyTorchDataset.__init__F   s    	
r   c                 x    SS K nUR                  U R                  5      UR                  U R                  5      S.$ )Nr   )r0   labels)torchTensorr0   r1   )r3   indexr8   s      r   __getitem__DummyTorchDataset.__getitem__K   s/    LL+ll4::.
 	
r   c                     U R                   $ r   )r2   )r3   s    r   __len__DummyTorchDataset.__len__R   s    xxr   r/   )returnN)__name__
__module____qualname____firstlineno__r4   r;   r>   __static_attributes__r   r   r   r,   r,   D   s    

r   r,   c                     [        XU5      $ r   )r,   r/   s      r   create_dummy_test_datasetrG   V   s    T#..r   r@   c                    [         R                  R                  U 5      (       dF  [        R                  " U5      n[        U S5       nUR                  UR                  5        S S S 5        [         R                  R                  U 5      n[         R                  R                  U 5      n[         R                  R                  [         R                  R                  U5      S   5      S   n[         R                  R                  Xg5      n[        R
                  " U 5      n	U	R                  US9  U$ ! , (       d  f       N= f)Nwbr   )path)r   rJ   existsrequestsgetopenwritecontentbasenamedirnamesplitextjointarfile
extractall)
fpathfurldstrf	file_nameroot_dirtarget_dir_nametarget_dir_pathts
             r   download_and_untarra   Z   s    77>>%  LL%!GGAII    'Iwwu%Hgg&&rww'7'7	'B1'EFqIOggll8=O 	UALLcL s   D11
D?c                  d   [         R                  " S5      u  pUR                  S5      n[        5       n[        5       nU H  nUR	                  5       nUR                  S5      nUS   nUR                  S5      nUS:X  d  US-
  [        U5      :  a  MT  XXS-
     n	UR                  U	5      n
XXU
 nXs;  a  [        5       X7'   X7   nUR                  U5        X;  a  [        5       XK'   XK   nUR                  UR                  SS	5      R                  S
S	5      R                  SS5      5        M     U$ )NzTgrep -rn "damo/" tests/  | grep -v "*.pyc" | grep -v "Binary file" | grep -v run.py 
:r   damo   ztests/ .py/.)
subprocessgetstatusoutputsplitr   stripfindlenrfindsetaddreplace)status_coderesultlines
test_casesmodel_caseslineelements	test_file	model_pos
left_quote
rquote_idx
model_name
model_info	case_infos                 r   get_case_model_infor   l   s)   $44^K LLEJ-Kzz|::c?QK	IIf%	?y1}D	9a-(
ZZ
+
J/
&$'EJ!*
z"(&)eK#+	h+33E4688?S8I	K' . r   c                    [        U5      n[        U5      nUR                  UR                  :w  a+  U b'  [        U  SUR                   SUR                   35        gUc  g[        U[        [
        [        [        R                  [        R                  [        R                  45      (       a  X:w  a  U b  [        U  SU SU 35        gg[        U[        [        R                  45      (       a3  [        R                  " XX4SS9(       d  U b  [        U  SU SU 35        gg[        U[        [        45      (       a  [!        U5      [!        U5      :w  a)  U b%  [        U  S[!        U5       S	[!        U5       35        g[#        [%        X5       VV	s/ s H  u  p['        S XX4S
9PM     sn	n5      (       d  U b  [        U  5        gg[        U[(        5      (       a  UR+                  5       n
UR+                  5       n[!        U
5      [!        U5      :w  a)  U b%  [        U  S[!        U
5       S	[!        U5       35        g[!        [-        U
5      [-        U5      -
  5      S:  a)  U b%  [        U  S[-        U
5      [-        U5      -
   35        g[#        U
 Vs/ s H  n['        S X   X,   X4S
9PM     sn5      (       d  U b  [        U  5        gg[        U[        R.                  5      (       a  [        R0                  " [        R2                  " US 5      [        R4                  U5      R7                  [        S9n[        R0                  " [        R2                  " US 5      [        R4                  U5      R7                  [        S9n[#        [        R                  " XX4SS9R9                  5       5      (       d  U b  [        U  5        ggU(       a  g[;        SU 35      es  sn	nf s  snf )Nz, type not equal:z and FTz, arg1:z, arg2:)rtolatol	equal_nanz, length is not equal:z, )r   r   z, key length is not equal:r   z, key diff:)dtypeztype not supported: )typerA   print
isinstancer   strboolnpbool_integerstr_floatfloatingisclosetuplelistrq   allzipcompare_arguments_nestedr   keysrs   ndarraywhereequalr   astypeflatten
ValueError)print_contentarg1arg2r   r   ignore_unknown_typetype1type2sub_arg1sub_arg2keys1keys2keys                r   r   r      su    JEJE~~'$ /!25>>2B%GWX |	D3T288RZZI	J	J<(wtfGD6BC	D5"++.	/	/zz$4dK(wtfGD6BC	D5$-	(	(t9D	!($o%;CI;bTT  +.d/
 +:&H )(4D*9
  
 ()	D'	"	"				u:U#($o%?E
|2cRWj\Z s5zCJ&'!+({3u:E
3J2KLM !
 !C )$)TYTF 
  
 ()	D"**	%	%xxt,bffd;BBBOxxt,bffd;BBBO

4D%)++2796 6 ()3E7;<<[
,
s   O
OaW  
import ast
import argparse
import pickle
import torch
from torch import distributed as dist
from modelscope.utils.torch_utils import get_dist_info
import {}

parser = argparse.ArgumentParser()
parser.add_argument('--save_all_ranks', type=ast.literal_eval, help='save all ranks results')
parser.add_argument('--save_file', type=str, help='save file')
parser.add_argument('--local_rank', type=int, default=0)
args = parser.parse_args()


def main():
    results = {}.{}({})  # module.func(params)
    if args.save_all_ranks:
        save_file = args.save_file + str(dist.get_rank())
        with open(save_file, 'wb') as f:
            pickle.dump(results, f)
    else:
        rank, _ = get_dist_info()
        if rank == 0:
            with open(args.save_file, 'wb') as f:
                pickle.dump(results, f)


if __name__ == '__main__':
    main()
c                   :    \ rS rSrSr  SS jr  SS jrS rSrg)	DistributedTestCasei  a  Distributed TestCase for test function with distributed mode.
Examples:
    >>> import torch
    >>> from torch import distributed as dist
    >>> from modelscope.utils.torch_utils import init_dist

    >>> def _test_func(*args, **kwargs):
    >>>     init_dist(launcher='pytorch')
    >>>     rank = dist.get_rank()
    >>>     if rank == 0:
    >>>         value = torch.tensor(1.0).cuda()
    >>>     else:
    >>>         value = torch.tensor(2.0).cuda()
    >>>     dist.all_reduce(value)
    >>>     return value.cpu().numpy()

    >>> class DistTest(DistributedTestCase):
    >>>     def test_function_dist(self):
    >>>         args = ()  # args should be python builtin type
    >>>         kwargs = {}  # kwargs should be python builtin type
    >>>         self.start(
    >>>             _test_func,
    >>>             num_gpus=2,
    >>>             assert_callback=lambda x: self.assertEqual(x, 3.0),
    >>>             *args,
    >>>             **kwargs,
    >>>         )
Nc                    UR                   R                  n[        R                  R	                  U5      u  p[        R                  R                  U
5      S   n
UR                  n/ nU HC  n[        U[        5      (       a  SR                  U5      nUR                  [        U5      5        ME     UR                  5        HK  u  p[        U[        5      (       a  SR                  U5      nUR                  SR                  X5      5        MM     SR                  U5      R                  S5      n[        R                  " SS9R                   n[        R                  " SS9R                   n[#        US5       n[%        S	R                  U5      5        [%        S
R                  U5      5        [&        R                  XX5      nUR)                  U5        S S S 5        / nU(       a0  [+        U5       H   nUR                  U[        U5      -   5        M"     OU/nU R-                  U R.                  U/U-   5        [0        R2                  " [        R4                  5      nSR                  UR7                  SS5      U	45      R9                  S5      US'   SUS'   SU< SU< 3nU< SU< SU< 3n[%        SU-  5        [:        R<                  " USUS9n/ nU H=  n[#        US5       nUR                  [>        R@                  " U5      5        S S S 5        M?     U(       d  US   nU(       a  U" U5        U RC                  USSR                  X5      S9  U$ ! , (       d  f       GN= f! , (       d  f       M  = f)Nr   z'{}'z{}={},ri   )suffixz.pklwzsave temporary run file to : {}zsave results to : {}rd   
PYTHONPATHrh   1NCCL_P2P_DISABLEz--save_all_ranks=z --save_file= zscript command: %sT)shellenvrbz.The test function ``{}`` in ``{}`` run failed!)msg)"__code__co_filenamer   rJ   rn   rS   rC   r   r   formatappenditemsrT   ro   tempfileNamedTemporaryFilenamerN   r   _DIST_SCRIPT_TEMPLATErO   range
addCleanup	clean_tmpcopydeepcopyr   rM   lstriprl   callpickleloadassertEqual)r3   dist_start_cmdfuncnum_gpusassert_callbacksave_all_ranksargskwargsscript_path
script_dirscript_name	func_namefunc_paramsargkvtmp_run_filetmp_res_filer[   run_file_contenttmp_res_filesitmp_envscript_params
script_cmdres
script_resres_files                               r   _startDistributedTestCase._start!  s    mm//"$''--"<
gg&&{3A6%%	C#s##s+s3x( 
 LLNDA!S!!__Q'w~~a34 #
 hh{+11#622%@EE22&AFF,$3::<HI(//=>4;;) BGG$% % 8_$$\CF%:; % *NM(FG--

+ #[[r*J7!99? 	 '*"#@N@LN#1<O
"Z/0ooj'B
%Hh%!!&++a.1 &% & #AJJ'@GG( 	 	) U %$8 &%s   8AM$&M%
M"%
M4	c           	          SSK Jn  [        R                  " [        R                  " 5       5      nSU;   a  UR                  S5      n	OS[        R                  X(U" 5       4-  n	U R                  " UU	UUUUS.UD6$ )Nrg   )_find_free_portr   zV%s -m torch.distributed.launch --nproc_per_node=%d --master_addr='%s' --master_port=%s)r   r   r   r   r   )	torch_utilsr   socketgethostbynamegethostnamepopsys
executabler   )
r3   r   r   r   r   r   r   r   ipr   s
             r   startDistributedTestCase.startj  s     	1!!&"4"4"67v%#ZZ(89NEHKX`fufwGxyN {{  *+)  	r   c                    U H{  n[         R                  R                  U5      (       d  M)  [         R                  R                  U5      (       a  [        R
                  " U5        Me  [         R                  " U5        M}     g r   )r   rJ   rK   isdirr   r   remove)r3   tmp_file_listfiles      r   r   DistributedTestCase.clean_tmp  sI    !Dww~~d##77==&&MM$'IIdO "r   r   )NF)	rA   rB   rC   rD   __doc__r   r   r   rE   r   r   r   r   r     s)    B  $#GX #"	0$r   r   )gMbP?g:0yE>T)2r   r   r   r   r   rl   r   rU   r   r!   collectionsr   collections.abcr   os.pathr   numpyr   rL   modelscope.hub.constantsr   modelscope.utils.import_utilsr   r   r
   r   r   rM   TEST_ACCESS_TOKEN1TEST_ACCESS_TOKEN2TEST_MODEL_CHINESE_NAMEr   hasattrnanr   r   r   r%   r'   r   r*   r,   rG   r   ra   r   r   r   TestCaser   r   r   r   <module>r	     s    	     
    # #    = M
 ZZ^^$>E ZZ^^$>E .  0(;r5VVBF7
# 
 $/C $H #("'15Q=h DE$(++ E$r   