
    9iv                     z   S SK r S SKrS SKrS SKrS SKJr  S SKrS SKrS SK	J	r	  S SK
Jr  S SKJr  S SKJr  S SKrS SKJr  S SKJrJrJrJrJrJrJrJrJrJrJr  S SKJ r J!r!  S S	K"J#r#  S S
K$J%r%  \" \&5      r'Sr(\'RS                  5       RT                  S   r+\#" 5       r,Sr-SS/r./ SQr/Sr0Sr1Sr2Sr3Sr4Sr5Sr6Sr7Sr8Sr9Sr:Sr;Sr<Sr=Sr>S r?S!r@S"rAS#rBS$ rC\C" 5       rD " S% S&\E5      rF " S' S(\E5      rG\G" 5       rHS)\IS\\J\4   S*S4S+ jrKS6S- jrLS7S. jrMS/ rNS0 rOSS,\,\04S1 jrPS8S2 jrQS9S3 jrR\SS4:X  a  \P" SS59rT\U" \T5        gg):    N)datetime)reduce)Path)Union)version)CustomDatasetsHeadsHooksLR_SchedulersMetricsModels
Optimizers	PipelinesPreprocessors
TaskModelsTrainers)FieldsTasks)get_modelscope_cache_dir)default_groupT   register_module
modelscope.)modelsmetrics	pipelinespreprocessorstrainers
msdatasets	exportersast_indexer
decoratorsexpressfrom_importsimportsfilepathmodelscope_pathr   md5indexfiles_mtimerequirementsmodule
class_name	group_keymodule_name
module_clsTEMPLATE_PATHzast_index_file.pyc                  p    [         R                  " S5      n U R                  [         R                  5        U $ )Nzmodelscope.ast)logging	getLoggersetLevelINFO)
ast_loggers    Z/var/www/html/land-doc-ocr/venv/lib/python3.13/site-packages/modelscope/utils/ast_utils.pyget_ast_loggerr:   ;   s+    ""#34J%    c            	          \ rS rSrSS jrS\S\4S jrS\R                  S\4S jr
S\\R                  S4   S\4S	 jrSS
\R                  S\S\4S jjrSS\R                  S\S\4S jjrS rS\\R                  S\4   4S jr  SS\\R                  S\4   S\S\S\4S jjrS\R                  S\4S jrS rS\S\S\4S jrS\S\4S jrS rSrg)AstScanningD   returnNc                 \    [        5       U l        [        5       U l        / U l        / U l        g N)dictresult_importresult_from_importresult_decoratorr$   selfs    r9   __init__AstScanning.__init__F   s%    !V"&& "r;   nodec                     [        U[        R                  5      =(       a    [        U[        R                  5      (       + $ rA   )
isinstanceastASTexpr_contextrG   rJ   s     r9   _is_sub_nodeAstScanning._is_sub_nodeL   s2    $''# N+5dC<L<L+M'M	Nr;   c                     UR                    He  n[        X5      nU R                  U5      (       a    g[        U[        [
        45      (       d  MC  U H  nU R                  U5      (       d  M      g   Mg     g)NFT)_fieldsgetattrrQ   rL   listtuple)rG   rJ   fieldattrvals        r9   _is_leafAstScanning._is_leafP   sb    \\E4'D  &&D4-00C((--$   " r;   strc                 Z    [         (       a   [        U5      R                  S:X  d  US:X  a  gg)NFunctionDefTF)SKIP_FUNCTION_SCANNINGtype__name__rP   s     r9   _skip_functionAstScanning._skip_function\   s'    !!Dz""m3t}7Lr;   nshow_offsetsc                 Z    U(       a  UR                   UR                  -   $ UR                  $ rA   )_attributesrT   )rG   re   rf   s      r9   rT   AstScanning._fieldsb   s"    ==199,,99r;   c                    [        5       n[        U[        R                  5      (       aV  [        5       nU R	                  XS9 H   nU R                  [        X5      US9nXdU'   M"     XC[        U5      R                  '   U$ U$ )Nrf   )	rB   rL   rM   rN   rT   _leafrU   ra   rb   )rG   rJ   rf   output
local_dictrX   field_outputs          r9   rl   AstScanning._leafh   s{    dCGG$$JdF#zzD(|  *  E$05! G +54:&&'MKr;   c                 \    [        5       U l        [        5       U l        / U l        / U l        g rA   )rB   rC   rD   rE   result_expressrF   s    r9   _refreshAstScanning._refreshu   s&    !V"&& " r;   c                 F    U R                  5         U R                  USSS9  g )Nz  F)indentrf   )_setup_globalscan_importrP   s     r9   scan_astAstScanning.scan_ast{   s#    d?r;   parent_node_namec                 	  ^ ^ Uc  U$ T R                  U5      (       a  T R                  UTS9$  SS[        [        R                  S [
        4   S[
        S[
        4U U4S jjjn[        5       n[        U5      R                  S:X  a`  [        US5      nUS	:  aN  SR                  S
/U-  5      n[        USS5        [        US5      nUc  [        USU5        O[        USXx-   5        T R                  UTS9 GHL  n	[        X5      n
U
/ :X  a  / XY'   GOKT R                  U5      (       a  M3  [        U
[        5      (       aZ  [!        U
5      S	:X  aK  [        U
S   [        R                  5      (       a)  T R                  U
S   5      (       a  U" U
S   5      nXU	'   O[        U
[        5      (       a  [        5       nU
 Hj  nU" U[        U5      R                  5      n[        U5      R                  nUS:X  d  US:X  d  US:X  d  US:X  d  MN  X;  a  / X'   X   R#                  U5        Ml     XU	'   O0[        U
[        R                  5      (       a  U" U
5      nXU	'   OXU	'   [        U5      R                  S:X  d  [        U5      R                  S:X  Ga  [        U5      R                  S:X  a  U	S:X  a  [        5       T R$                  XY   '   U	S:X  as  [        XY   S   [        5      (       a<  / nXY   S    H  nUS   S   nUR#                  U5        M     UT R$                  US   '   OXY   S   S   nU/T R$                  US   '   [        U5      R                  S:X  aU  XY   S   n[        U[        5      (       a"  U H  nUS   T R&                  US   S   '   M     OUT R&                  XY   S   S   '   SU	:X  aE  U
/ :w  a?  U
 H  n[        U[(        UR*                  5        M      T R,                  R/                  U
5        U
/ :w  d  GM  [        U
5      R                  S:X  d  GM(  US:X  d  GM1  T R0                  R#                  U
5        GMO     [2        T R&                  [4        T R$                  [6        T R,                  [8        T R0                  0$ )Nrk    elr{   r?   c                 &   > TR                  U TUS9$ )N)rf   r{   )rx   )r~   r{   rG   rf   s     r9   _scan_import-AstScanning.scan_import.<locals>._scan_import   s$    ''!-%5 ( 7 7r;   
ImportFromlevelr   r   r   r-   Importnamesaliasnamedecorator_listCallExpr)r}   )r[   rl   r   rM   rN   r]   rB   ra   rb   rU   joinsetattrrT   rc   rL   rV   lenappendrD   rC   
CLASS_NAMEr   rE   extendrr   
IMPORT_KEYFROM_IMPORT_KEYDECORATOR_KEYEXPRESS_KEY)rG   rJ   rf   r{   r   outputsr   
path_levelr0   rX   rY   	local_outel_dictr~   r   rm   	item_nameitem
local_name
final_dicts   ` `                 r9   rx   AstScanning.scan_import   sM    <K]]4  ::d:>> 687swwc'9!: 7/27<?7 7 fGDz""l2g.A:!##!7JD'1-")$"9K"*h
;h
0HIdFt+2:%'GN(()9:: t,,Ta$T!Wcgg66--Q00 ,T!W 5I%.ENd++"fG"$0T"X5F5F$G	#Bx00 H,0D#3|#C#3x#?#202#M00; # &-ENcgg..)$/F%+EN%)ENJ''83:..,>Dz**l: H,FJfD33GNC G+)'.*A4HH,.	,3N7,CD15gv1FJ$-$4$4Z$@ -D :C !% 7 7$+H$5!7 .5^G-DV-L
$.N" 7 78I J Dz**h6%,^G%<
%j$77(2?CG} !% 2 24=4:4< != )3
 <F !..w~g/F0608 9 $u, $j$))< !%))0062:$#(f#%)9V)C''..t4E GJ D..!8!8t44T00	 r;   c                   ^ S[         R                  S[        4S jmS[        S[        4U4S jjnS[         R                  S[        4U4S jjnT" UR                  5      nU" UR
                  5      nU" UR                  5      nXEU4$ )NrJ   r?   c                     Su  pn[        U 5      R                  S:X  a$  [        U S5      n[        US5      n[        U S5      n[        U 5      R                  S:X  a  [        U S5      nX#4$ )N)NNN	AttributevalueidrY   Name)ra   rb   rU   )rJ   r   r   rY   s       r9   _get_attribute_item9AstScanning._parse_decorator.<locals>._get_attribute_item   sf    .OEtDz""k1g.UD)tV,Dz""f,T4(8Or;   nodesc                 ,  > / nU  H  n[        U5      R                  S:X  a  UR                  UR                  S 45        M;  [        U5      R                  S:X  a  UR                  UR                  S 45        Ms  UR                  T" U5      5        M     U$ )NStrConstant)ra   rb   r   sr   )r   resultrJ   r   s      r9   _get_args_name4AstScanning._parse_decorator.<locals>._get_args_name   sx    F:&&%/MM4664.1$Z((J6MM4::t"45MM"5d";<  Mr;   c                   > / nU  H  n[        U5      R                  S:X  d  M  [        US5      n[        U5      R                  S:X  a*  UR                  [        US5      UR                  S 45        Mm  [        U5      R                  S:X  a*  UR                  [        US5      UR
                  S 45        M  UR                  [        US5      4T" U5      -   5        M     U$ )Nkeywordr   r   argr   )ra   rb   rU   r   r   r   )r   r   rJ   attribute_noder   s       r9   _get_keyword_name7AstScanning._parse_decorator.<locals>._get_keyword_name   s    F:&&)3%,T7%;NN+44=wt/4(67E7G7G'O Pn-66*D$T51>3G3GNP wtU';&>(;N(K'L M  Mr;   )rM   rN   rW   rV   funcargskeywords)rG   rJ   r   r   	functions	args_listkeyword_listr   s          @r9   _parse_decoratorAstScanning._parse_decorator   s{    	cgg 	% 			$ 		4 			SWW 	 	  (		2	"499-	(7\11r;   c                     Uc  g US:X  a  [         $ UR                  S5      n[        U5      S:X  a  U$ [        [	        US   5      US   5      $ )Nr   r   r   r   )r   splitr   rU   eval)rG   key_item
split_lists      r9   _get_registry_valueAstScanning._get_registry_value  sR    &  ^^C(
z?aO4
1.
1>>r;   parsed_inputr.   c                    Uu  p4n[         US   :w  a  gUS   /n[        U5      S:X  a$  [        U5      S:X  a  UR                  [        5        [        U5      S:X  a   [        U5      S:X  a  UR                  U5        [        U5      S:  aS  [        U5      S:X  aD  SnU H(  nUu  pnU	[        :X  d  M  UR                  X45        UnM*     Ub  UR                  U5        [        U5      S:X  a  UR                  [        5        U H(  nUu  pnU	[        :X  a  U
nM  UR                  X45        M*     U H}  nUc  UR                  S5        M  US   c  UR                  US   5        M5  [        U[        5      (       a  UR                  U5        M]  UR                  SR                  U5      5        M     US   U R                  US   5      U R                  US   5      4$ )zzformat registry information to a tuple indexer

Return:
    tuple: (MODELS, Tasks.text-classification, Models.structbert)
r   Nr   r      )REGISTER_MODULEr   r   r   	GROUP_KEYremove
MODULE_CLSrL   r]   r   r   )rG   r   r.   r   r   r   rm   remove_group_itemr   keyr   rY   s               r9   _registry_indexerAstScanning._registry_indexer!  s    .:*	l il*A,y>Q3|#4#9]+|!c)n&9Z(|q S^q%8 $$"&4)#$$d\2(,%	 %
 !,##$56y>Q]+ D"OCtj !
  $. ! D|d#ad1g&D#&&d#chhtn-  q	433F1I>((35 	5r;   r   c                 .   / nU H  n[        U5      R                  S:w  a  M  [        U[        S5      n[        US5      n[        USS5      [        :w  a  MR  U R                  U5      nU R                  Xd5      nSULd  M{  UR                  U5        M     U$ )zparse the AST nodes of decorators object to registry indexer

Args:
    nodes (list): list of AST decorator nodes

Returns:
    list: list of registry indexer
r   Nr   rY   )ra   rb   rU   r   r   r   r   r   )rG   r   resultsrJ   r.   r   parse_outputr*   s           r9   parse_decoratorsAstScanning.parse_decoratorsU  s     DDz""f, z48J4(DtVT*o=006L**<DE5 u%  r;   c                    U R                  5         [        USSS9 nUR                  5       nS S S 5        SR                  W5      n[        R
                  " U5      nU R                  USS9nU R                  U[           5      U[        '   U R                  U[           5      U[        '   U[           R                  U[           5        U$ ! , (       d  f       N= f)Nrutf8)encodingr}   Frk   )rs   open	readlinesr   rM   parserx   r   r   r   r   )rG   filecodedatarJ   rm   s         r9   generate_astAstScanning.generate_astn  s    $f->>#D .wwt}yy!!$U!; $ 5 5f]6K L}"33F;4GH{}$$VK%89 .-s   C
C)r$   rE   rr   rD   rC   r?   N)T)Tr}   )rb   
__module____qualname____firstlineno__rH   objectboolrQ   rM   rN   r[   r   rc   rW   rT   r]   rl   rs   ry   rx   r   r   r   rV   r   r   __static_attributes__ r;   r9   r=   r=   D   sL   N ND N
SWW 
 
5%#8 T    #''   !@U377D##56 @ " "	hCGGT3&'h h 	h
 
hT*2SWW *2 *2X
?25e 25 25 25hd t 2
r;   r=   c                       \ rS rSrSS jr SS\S\S\4S jjrS rS	\S\	4S
 jr
SS jrSS jrS rS rS rS rS\\4S jr\\S4S jrSrg)FilesAstScanningi{  r?   Nc                 >    [        5       U l        / U l        / U l        g rA   )r=   	astScaner	file_dirsrequirement_dirsrF   s    r9   rH   FilesAstScanning.__init__}  s    $ "r;   import_packagecurrent_pathc                    UR                  [        S   5      (       a0  [        S-   SR                  UR	                  S5      SS 5      -   S-   $ UR                  [        S   5      (       as  UR	                  S5      nUR	                  S5      nSn[        U5       H  u  pgUS:w  d  M  Un  O   USU*  nUR                  UWS 5        SSR                  U5      -   S-   $ U$ )zl
Args:
    import_package (str): relative import or abs import
    current_path (str): path/to/current/file
r   /r   r   N.pyr}   )
startswithIGNORED_PACKAGESMODELSCOPE_PATHr   r   	enumerater   )	rG   r   r   current_path_listimport_package_listr   r*   r   abs_path_lists	            r9   _parse_import_path#FilesAstScanning._parse_import_path  s     $$%5a%899"S(388$$S)!"-,/ /167 7&&'7':;; , 2 23 7"0"6"6s";E()<=2:!E  >
 .a7M  !4UV!<=-00588r;   c                     g rA   r   )rG   import_abs_paths     r9   _traversal_import"FilesAstScanning._traversal_import  s     	r;   scan_resultc                 $   / nUR                  [        U[           R                  5       5      5        UR                  [        U[           R                  5       5      5        [        U5       H9  u  p4SUR                  S5      S   :X  a  SX#'   M#  UR                  S5      S   X#'   M;     [        5       nU H9  n[         H,  nUR                  U5      (       d  M  UR                  U5        M.     M;     [        [        U5      [        U5      -
  5      $ )zparse import and from import dicts to a third party package list

Args:
    scan_result (dict): including the import and from import result

Returns:
    list: a list of package ignored 'modelscope' and relative path import
r}   r   r   )r   rV   r   keysr   r   r   setr   r   add)rG   r   rm   r*   r   ignoredignored_packages          r9   parse_importFilesAstScanning.parse_import  s     d;z2779:;d;7<<>?@ %V,KETZZ_Q'' # $

3 2	 - %D#3???33KK% $4  CK#g,.//r;   c                    / U l         Ub  [        U5      S:X  a  U R                  XS9  g U HW  n[        R                  R                  X5      n[        R                  R                  U5      (       d  MH  U R                  XSS9  MY     g )Nr   include_init)r   r   _traversal_filesospathr   isdir)rG   r  check_sub_dirr  r   sub_dirs         r9   traversal_files FilesAstScanning.traversal_files  sl     C$6!$;!!$!B%'',,t277==))))')M &r;   c                    [         R                  " U5      nU GHm  nUR                  S:X  a	  U(       d  M  UR                  R                  S5      (       a  UR                  S:w  d@  UR                  R	                  S5      (       d   UR                  R	                  S5      (       a  M  UR                  5       (       a  U R                  UR                  US9  M  UR                  5       (       aH  UR                  R	                  S5      (       a(  U R                  R                  UR                  5        GM  UR                  5       (       d  GM5  SUR                  ;   d  GMH  U R                  R                  UR                  5        GMp     g )Nz__init__.py__z.jsonz.mdr
  r   requirement)r  scandirr   r   endswithis_dirr  r  is_filer   r   r   )rG   r  r  dir_listr   s        r9   r  !FilesAstScanning._traversal_files  s   ::d#DyyM),))&&t,,ii=0TYY5G5G6 6"&))"4"4U";";{{}}%%diil%KDII$6$6u$=$=%%dii0MTYY$>%%,,TYY7 r;   c                     U R                   R                  U5      nU R                  U5      nU[           U4$ ! [         a  n[        R                  " UR
                  5      n[        SU SUS   R                   SUS   R                   SUS   R                   S[        U5      R                   SU SU S	35      eS nAff = f)
NzDuring ast indexing the file z', a related error excepted in the file z
 at line: z: "z" with error msg: "z: z'", please double check the origin file z- to see whether the file is correctly edited.)r   r   	Exception	traceback
extract_tb__traceback__filenamelinenolinera   rb   r  r   )rG   r   rm   edetailimport_lists         r9   _get_single_file_scan_result-FilesAstScanning._get_single_file_scan_result  s    		A^^006F ''/m$k11  	A))!//:F/v 6%bz223:":$$%S(9 :G$$%Rs*QRVQW X?@A A	As   9 
CA;B>>Cc           	          [        5       nU H=  nX   [            H+  n[        U[        X   [           [        X   [           0X$'   M-     M?     U$ rA   )rB   r   FILE_NAME_KEYr   
MODULE_KEY)rG   forward_indexinverted_indexr*   r   s        r9   _inverted_index FilesAstScanning._inverted_index  sV    "E%,];!5 4Z @ 4Z @($ < # r;   c                 p    [        5       nUR                  5        H  u  p4U[           X$[           '   M     U$ rA   )rB   itemsr   r-  )rG   r.  module_importr*   
value_dicts        r9   _module_importFilesAstScanning._module_import  s7    !.!4!4!6E4>z4JMZ01 "7r;   c                 *    SU;   a  US	 SU;   a  US	 U$ )N)
OPTIMIZERSdefaultr   )LR_SCHEDULERr:  r   r   )rG   r/  s     r9   _ignore_useless_keys%FilesAstScanning._ignore_useless_keys  s*    ,>@A..@BCr;   c                    [         R                   " 5       nUb  Xl        OU R                  X#5        [        R	                  SU SU 35        [        5       nU R                   Ho  nXfR                  S5      S nUR                  [        R                  S5      R                  SS5      nU R                  U5      u  p[        U	[        U
[        U0XV'   Mq     U R                  U5      nU R                  U5      nU R!                  U5      n["        U[$        U0n[        R	                  S['        U5       S	[         R                   " 5       U-
   S
35        U$ )a  the entry method of the ast scan method

Args:
    target_file_list can override the dir and folders combine
    target_dir (str, optional): the absolute path of the target directory to be scanned. Defaults to None.
    target_folder (list, optional): the list of
    sub-folders to be scanned in the target folder.
    Defaults to SCAN_SUB_FOLDERS.

Returns:
    dict: indexer of registry
NzAST-Scanning the path "z!" with the following sub folders r   r   r   r}   zScanning done! A number of z. components indexed or updated! Time consumed r   )timer   r  loggerinforB   rfindreplaceospsepr)  r   r   r-  r0  r<  r6  	INDEX_KEYREQUIREMENT_KEYr   )rG   target_file_list
target_dirtarget_foldersstartr   r   r'   r0   r   r(  inverted_index_with_resultsr4  r*   s                 r9   get_files_scan_results'FilesAstScanning.get_files_scan_results  sP     		'-N  <%j\1RSaRbc	
 NNDJJ|456H"**377C8@@KK*.*K*K+'N ~KKFL # '+&:&:6&B#&*&?&?'')#++F32]
 	)#.I*J)K L<<@IIK<M;NaQ	
 r;   c                    / U l         U(       a  [        U[        5      (       a  X0l         OU R                  X5        / n[	        5       nU R                    H7  n[
        R                  R                  U5      nUR                  U5        XuU'   M9     [        S US5      n[        R                  " UR                  5       5      n	U	R                  5       U4$ )Nc                 0    [        U 5      [        U5      -   $ rA   )r]   )xys     r9   <lambda>2FilesAstScanning.files_mtime_md5.<locals>.<lambda>?  s    Q#a&r;   r}   )r   rL   rV   r  rB   r  r  getmtimer   r   hashlibr)   encode	hexdigest)
rG   target_pathtarget_subfolder	file_listr+   files_mtime_dictr   mtime
result_strr)   s
             r9   files_mtime_md5 FilesAstScanning.files_mtime_md50  s     It44&N  ?6NNDGG$$T*Eu%%*T" # 8+rJ
kk*++-.}} 000r;   )r   r   r   r   rA   NFF)rb   r   r   r   rH   r]   r   r   rB   rV   r  r  r  r)  r0  r6  r<  r   SCAN_SUB_FOLDERSrM  r_  r   r   r;   r9   r   r   {  s    # 04 +. ), 8; 40 0 06N8 2	 15*9.>0f %4)9"&1r;   r   objr?   c                 0   [         R                  R                  U5      nU(       a9  [         R                  R                  U5      (       d  [         R                  " USS9  [        US5       nUR                  U 5        SSS5        g! , (       d  f       g= f)zWrite data to a given ``filepath`` with 'wb' mode.

Note:
    ``write`` will create a directory if the directory of ``filepath``
    does not exist.

Args:
    obj (bytes): Data to be written.
    filepath (str or Path): Path to write data.
T)exist_okwbN)r  r  dirnameexistsmakedirsr   write)rd  r'   rh  fs       r9   ensure_writerm  G  s\     ggooh'Grww~~g..
Gd+	h		 
		s   ,B
BFc                 ~   U [            R                  5        VVs0 s H  u  pE[        U5      U_M     snnU [         '   SSKJn  X`[
        '   [        R                  US9u  U [        '   U [        '   [        R                  5       U [        '   [        R                  " U 5      nU(       a(  UR                  [        R                  5       [         5      n[#        UR%                  5       U5        U [            R                  5        VVs0 s H  u  pE[&        R(                  " U5      U_M     snnU [         '   g s  snnf s  snnf )Nr   __version__r[  )rF  r3  r]   modelscope.versionrp  VERSION_KEYfile_scannerr_  MD5_KEYFILES_MTIME_KEYr   as_posixMODELSCOPE_PATH_KEYjsondumpsrC  r2   rm  rW  rM   literal_eval)r*   	file_pathr[  with_templatekvrp  
json_indexs           r9   _save_indexr  Z  s   .3I.>.D.D.FG.FdaA	.FGE).$+-9-I-I .J .*E'NE/*!0!9!9!;E
E"J''(@(@(B(57
""$i0 )$**,,DA 	Q,E) Hs   D3#D9c                    [        U S5       nUR                  5       nS S S 5        U(       a6  WR                  5       R                  [        [
        R                  5       5      n[        R                  " W5      nU[           R                  5        VVs0 s H  u  pV[        R                  " U5      U_M     snnU[        '   U$ ! , (       d  f       N= fs  snnf )Nrb)r   readdecoderC  r2   r   rw  ry  loadsrF  r3  rM   r{  )r|  r}  rl  bytes_indexwrapped_indexr~  r  s          r9   _load_indexr  m  s    	i	!ffh 
!((*22=3B3K3K3MOJJ{+M "),224 4DA 	Q4 M)  
	 s   C#C
Cc                 $   U [            n[        [        U5      [        U5      -
  5      n[        [        U5      [        U5      -
  5      n/ nU H'  nXd;  d  M
  X&   X   :w  d  M  UR                  U5        M)     UR	                  U5        UR	                  U5        [        U5      S:  a  / n/ nU [            HO  n	U [           U	   [           U;   d  M  UR                  U	5        UR                  U [           U	   [           5        MQ     U H  n	U [           U		 M     U H  n	X[           ;   d  M  U [           U		 M     [        R                  U5      n
U [           R                  U
[           5        U [           R                  U
[           5        g )Nr   )rv  rV   r  r   r   r   rF  r,  r-  rG  rt  rM  update)r*   r+   origin_files_mtime	new_filesremoved_filesupdated_filesr   remove_index_keysremove_requirement_keysr   updated_indexs              r9   _update_indexr  |  s|   /S%,>(??@I/03{3CCDMM"$#)[->>  & # '# =A"$#CY$]3}D!((-'..)$S)*57 $
 %Ci % %*CO,,/*3/ +
 !77FM	)M)45	/!!-"@Ar;   c                     [        [        [        R                  " [        R
                  S5      R                  5       5      5      n Sn[        [        [        R                  " 5       R                  5       5      5      nXU-   :  a  gg)Nz%Y-%m-%d %H:%M:%Si3TF)introundr   strptimer   __release_datetime__	timestampnow)release_timestampSECONDS_PER_YEARcurrent_timestamps      r9   __is_develop_modelr    sp    g::133<9;	@A *E(,,.":":"<=>/???r;   c                    [         R                  " SU5      n[         R                  " SU5      n[         R                  R                  XE5      nSnU(       a,  [        R                  S5        [        R                  U 5      nU$ [        5       (       Ga1  [        R                  SU 35        [         R                  R                  U5      (       a|  [        U5      n[        R                  U S9u  pUnSSKJn  U[           U:X  a  U[           U	:w  d  U[           U:w  a,  [        R                  S	5        [!        Xz5        [#        XvU 5        O:[        R                  S
U S35        [        R                  U 5      n[#        XvU 5        [        R                  SU[            SU[            S[%        U[&           5       S35        U$ [)        5       nU$ )a*  get the index from scan results or cache

Args:
    file_list: load indexer only from the file lists if provided, default as None
    force_rebuild: If set true, rebuild and load index, default as False,
    indexer_file_dir: The dir where the indexer file saved, default as INDEXER_FILE_DIR
    indexer_file: The indexer file name, default as INDEXER_FILE
Returns:
    dict: the index information for all registered modules, including key:
    index, requirements, files last modified time, modelscope home path,
    version and md5, the detail is shown below example: {
        'index': {
            ('MODELS', 'nlp', 'bert'):{
                'filepath' : 'path/to/the/registered/model', 'imports':
                ['os', 'torch', 'typing'] 'module':
                'modelscope.models.nlp.bert'
            },
            ...
        }, 'requirements': {
            'modelscope.models.nlp.bert': ['os', 'torch', 'typing'],
            'modelscope.models.nlp.structbert': ['os', 'torch', 'typing'],
            ...
        }, 'files_mtime' : {
            '/User/Path/To/Your/Modelscope/modelscope/preprocessors/nlp/text_generation_preprocessor.py':
            16554565445, ...
        },'version': '0.2.3', 'md5': '8616924970fe6bc119d1562832625612',
        'modelscope_path': '/User/Path/To/Your/Modelscope'
    }
MODELSCOPE_CACHEMODELSCOPE_INDEX_FILENz4Force rebuilding ast index from scanning every file!zLoading ast index from rq  r   ro  z}Updating the files for the changes of local files, first time updating will take longer time! Please wait till updating done!zNo valid ast index found from z$, generating ast index from scratch!z,Loading done! Current index file version is z, with md5 z and a total number of z components indexed)r  getenvr  r   r@  rA  rt  rM  r  ri  r  r_  rr  rp  rs  ru  r  r  r   rF  load_from_prebuilt)r[  force_rebuildindexer_file_dirindexer_file	cache_dir
index_filer|  r*   r  r)   r+   rp  s               r9   
load_indexr    s   H 		,.>?I2LAJY3IEJK33I> -i[9:77>>)$$'	2M+;;#  <  %C!E6k*k9%g.#5!+.+=a e1Ei8KK0;_` !77E)4:5;M:N Og''>5#$%%8:	; L #$Lr;   c                 
   U cE  [         R                  5       R                  S   n[        R                  R                  U[        5      n [        R                  R                  U 5      (       a  [        U SS9nU$ [        5       nU$ )Nr   Tr}  )
presolveparentsr  r  r   TEMPLATE_FILEri  r  generate_ast_template)r|  
local_pathr*   s      r9   r  r    sg    YY[((+
GGLL];		ww~~i  IT: L &'Lr;   c                    [        US9nU cE  [        R                  5       R                  S   n[        R
                  R                  U[        5      n [        X SS9  [        R
                  R                  U 5      (       d  [        S5      eU$ )Nr  r   Tr  z;The index file is not create correctly, please double check)r  r  r  r  r  r  r   r  r  ri  r  )r|  r  r*   r  s       r9   r  r    sp    ]3EYY[((+
GGLL];	577>>)$$IK 	KLr;   __main__r  ra  rb  rA   )NT)VrM   rV  r4   r  os.pathr  rD  r?  r   r   	functoolsr   pathlibr   typingr   ry  r   r   modelscope.metainfor   r	   r
   r   r   r   r   r   r   r   r   modelscope.utils.constantr   r   modelscope.utils.file_utilsr   modelscope.utils.registryr   __file__r  r`   r  r  r   INDEXER_FILE_DIRr   r   rc  INDEXER_FILEr   r   r   r   r,  rx  rs  ru  rF  rv  rG  r-  r   r   MODULE_NAMEr   r2   r  r:   r@  r   r=   r   rt  bytesr]   rm  r  r  r  r  r  r  r  rb   r*   printr   r;   r9   <module>r     s      	         F F F F 4 @ 3N  ))+%%a(+- # #&    
' 
	 

	
# 
	t& tn	F1v F1R  !e uS$Y'7 D &&BD
 %	Nb	 zT*E	%L r;   