
    9ia              %       \   S SK r S SKrS SKrS SKrS SKJr  S SKJr  S SKJ	r	  S SK
JrJrJrJrJr  S SKJrJrJrJrJrJr  S SKJr  S SKJr  S S	KJr  S
SKJrJr  S
SK J!r!  S
SK"J#r#  S
SK$J%r%  S
SK&J'r'J(r(J)r)  S
SK*J+r+  S
SK,J-r-J.r.J/r/J0r0  \" 5       r1SSSSSSSSSSSSS\SS4S\2S\\2   S\\2\	S4   S\\\\24      S\\3   S\\   S\\\2\\2   4      S\\\2\\2   4      S\\2   S\\\\2   \24      S\\\\2   \24      S\\4   S\2S \\2   S!\\3   S"\\\!      S#\24"S$ jjr5\SSSSSSSSSSS%4S&\2S\\2   S\\2\	S4   S\\2   S\\\\24      S\\3   S\\   S\\\2\\2   4      S\\\2\\2   4      S\\\\2   \24      S\\\\2   \24      S!\\3   S\4S#\24S' jjr6S\SSSSSSSSSS%SS(.S\2S \\2   S\\2   S\\2\	S4   S\\\\24      S\\3   S\\   S\\\2\\2   4      S\\\2\\2   4      S\\2   S\\\\2   \24      S\\\\2   \24      S\4S"\\\!      4S) jjr7S* r8S+\24S, jr9S-\\2\\2   4   4S. jr:S-\\2   4S/ jr;S\SSSSSS%SS4
S0\\2   S1\+S2\2S\2S3\S4\2S5\2S \\2   S\\2   S\\   S\\\2\\2   4      S\\\2\\2   4      S\\\\2   \24      S\\\\2   \24      S\4S6\\2   S"\\\!      4"S7 jjr<g)8    N)nullcontext)	CookieJar)Path)DictListOptionalTypeUnion)DEFAULT_DATASET_REVISIONDEFAULT_MODEL_REVISIONINTRA_CLOUD_ACCELERATIONREPO_TYPE_DATASETREPO_TYPE_MODELREPO_TYPE_SUPPORT)get_modelscope_cache_dir)
get_logger)thread_executor   )HubApiModelScopeConfig)ProgressCallback)DEFAULT_MAX_WORKERS)InvalidParameter)$create_temporary_directory_and_cachedownload_fileget_file_download_url)ModelFileSystemCache)get_model_masked_directorymodel_id_to_group_owner_name	strtoboolweak_file_lockFmodel_idrevision	cache_dir
user_agentlocal_files_onlycookiesignore_file_patternallow_file_pattern	local_dirallow_patternsignore_patternsmax_workersrepo_id	repo_typeenable_file_lockprogress_callbacksreturnc                    U=(       d    U nU(       d  [        S5      eU[        ;  a  [        SU S[         35      eU=(       d    [        nUc  U[        :X  a  [        O[
        nUc)  [        [        R                  R                  SS5      5      nU(       a  Ub  UO	[        5       n[        R                  " [        R                  R                  US5      SS	9  [        R                  R                  USUR                  S
S5      5      n[        U5      nO
[!        5       nU   [#        UUUUUUUUUUU
U	UUS9sSSS5        $ ! , (       d  f       g= f)a  Download all files of a repo.
Downloads a whole snapshot of a repo's files at the specified revision. This
is useful when you want all files from a repo, because you don't know which
ones you will need a priori. All files are nested inside a folder in order
to keep their actual filename relative to that folder.

An alternative would be to just clone a repo but this would require that the
user always has git and git-lfs installed, and properly configured.

Args:
    repo_id (str): A user or an organization name and a repo name separated by a `/`.
    model_id (str): A user or an organization name and a model name separated by a `/`.
        if `repo_id` is provided, `model_id` will be ignored.
    repo_type (str, optional): The type of the repo, either 'model' or 'dataset'.
    revision (str, optional): An optional Git revision id which can be a branch name, a tag, or a
        commit hash. NOTE: currently only branch and tag name is supported
    cache_dir (str, Path, optional): Path to the folder where cached files are stored, model will
        be save as cache_dir/model_id/THE_MODEL_FILES.
    user_agent (str, dict, optional): The user-agent info in the form of a dictionary or a string.
    local_files_only (bool, optional): If `True`, avoid downloading the file and return the path to the
        local cached file if it exists.
    cookies (CookieJar, optional): The cookie of the request, default None.
    ignore_file_pattern (`str` or `List`, *optional*, default to `None`):
        Any file pattern to be ignored in downloading, like exact file names or file extensions.
    allow_file_pattern (`str` or `List`, *optional*, default to `None`):
        Any file pattern to be downloading, like exact file names or file extensions.
    local_dir (str, optional): Specific local directory path to which the file will be downloaded.
    allow_patterns (`str` or `List`, *optional*, default to `None`):
        If provided, only files matching at least one pattern are downloaded, priority over allow_file_pattern.
        For hugging-face compatibility.
    ignore_patterns (`str` or `List`, *optional*, default to `None`):
        If provided, files matching any of the patterns are not downloaded, priority over ignore_file_pattern.
        For hugging-face compatibility.
    max_workers (`int`): The maximum number of workers to download files, default 8.
    enable_file_lock (`bool`): Enable file lock, this is useful in multiprocessing downloading, default `True`.
        If you find something wrong with file lock and have a problem modifying your code,
        change `MODELSCOPE_HUB_FILE_LOCK` env to `false`.
    progress_callbacks (`List[Type[ProgressCallback]]`, **optional**, default to `None`):
        progress callbacks to track the download progress.
Raises:
    ValueError: the value details.

Returns:
    str: Local folder path (string) of repo snapshot

Note:
    Raises the following errors:
    - [`EnvironmentError`](https://docs.python.org/3/library/exceptions.html#EnvironmentError)
    if `use_auth_token=True` and the token cannot be found.
    - [`OSError`](https://docs.python.org/3/library/exceptions.html#OSError) if
    ETag cannot be determined.
    - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)
    if some parameter value is invalid
z*Please provide a valid model_id or repo_idInvalid repo type: , only support: NMODELSCOPE_HUB_FILE_LOCKtrue.lockTexist_ok/___)r/   r#   r$   r%   r&   r'   r(   r)   r*   r,   r+   r-   r1   )
ValueErrorr   r   r   r   r   r    osenvirongetr   makedirspathjoinreplacer!   r   _snapshot_download)r"   r#   r$   r%   r&   r'   r(   r)   r*   r+   r,   r-   r.   r/   r0   r1   system_cache	lock_filecontexts                      `/var/www/html/land-doc-ocr/venv/lib/python3.13/site-packages/modelscope/hub/snapshot_download.pysnapshot_downloadrJ   "   s9   R !GEFF))!),<=N<OP
 	
 4!4K/8<M/M+Si$JJNN5v>@ $-$9y?W @

BGGLLw7$GGGLLw!(e!<>	 +-	!!- 31+)#13 
s   #E
E   
dataset_idc                    Uc)  [        [        R                  R                  SS5      5      nU(       a  Ub  UO	[	        5       n[        R
                  " [        R                  R                  US5      SS9  [        R                  R                  USU R                  SS5      5      n[        U5      nO
[        5       nU   [        U [        UUUUUUUUU
U	US	9sSSS5        $ ! , (       d  f       g= f)
a  Download raw files of a dataset.
Downloads all files at the specified revision. This
is useful when you want all files from a dataset, because you don't know which
ones you will need a priori. All files are nested inside a folder in order
to keep their actual filename relative to that folder.

An alternative would be to just clone a dataset but this would require that the
user always has git and git-lfs installed, and properly configured.

Args:
    dataset_id (str): A user or an organization name and a dataset name separated by a `/`.
    revision (str, optional): An optional Git revision id which can be a branch name, a tag, or a
        commit hash. NOTE: currently only branch and tag name is supported
    cache_dir (str, Path, optional): Path to the folder where cached files are stored, dataset will
        be save as cache_dir/dataset_id/THE_DATASET_FILES.
    local_dir (str, optional): Specific local directory path to which the file will be downloaded.
    user_agent (str, dict, optional): The user-agent info in the form of a dictionary or a string.
    local_files_only (bool, optional): If `True`, avoid downloading the file and return the path to the
        local cached file if it exists.
    cookies (CookieJar, optional): The cookie of the request, default None.
    ignore_file_pattern (`str` or `List`, *optional*, default to `None`):
        Any file pattern to be ignored in downloading, like exact file names or file extensions.
        Use regression is deprecated.
    allow_file_pattern (`str` or `List`, *optional*, default to `None`):
        Any file pattern to be downloading, like exact file names or file extensions.
    allow_patterns (`str` or `List`, *optional*, default to `None`):
        If provided, only files matching at least one pattern are downloaded, priority over allow_file_pattern.
        For hugging-face compatibility.
    ignore_patterns (`str` or `List`, *optional*, default to `None`):
        If provided, files matching any of the patterns are not downloaded, priority over ignore_file_pattern.
        For hugging-face compatibility.
    enable_file_lock (`bool`): Enable file lock, this is useful in multiprocessing downloading, default `True`.
        If you find something wrong with file lock and have a problem modifying your code,
        change `MODELSCOPE_HUB_FILE_LOCK` env to `false`.
    max_workers (`int`): The maximum number of workers to download files, default 8.
Raises:
    ValueError: the value details.

Returns:
    str: Local folder path (string) of repo snapshot

Note:
    Raises the following errors:
    - [`EnvironmentError`](https://docs.python.org/3/library/exceptions.html#EnvironmentError)
    if `use_auth_token=True` and the token cannot be found.
    - [`OSError`](https://docs.python.org/3/library/exceptions.html#OSError) if
    ETag cannot be determined.
    - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)
    if some parameter value is invalid
Nr6   r7   r8   Tr9   r;   r<   )r/   r#   r$   r%   r&   r'   r(   r)   r*   r,   r+   r-   )r    r>   r?   r@   r   rA   rB   rC   rD   r!   r   rE   r   )rL   r#   r$   r*   r%   r&   r'   r(   r)   r+   r,   r0   r-   rF   rG   rH   s                   rI   dataset_snapshot_downloadrN      s    B $JJNN5v>@ $-$9y?W @

BGGLLw7$GGGLLw!+!3!3C!?A	 +-	!'!- 31+)#% 
s   ?C""
C0)r/   r#   r$   r%   r&   r'   r(   r)   r*   r+   r,   r-   r1   c                	   U(       d  [         nU[        ;  a  [        SU< S[        < 35      e[        X	X1S9u  pUb  UO	[	        5       nU(       aL  [        UR                  5      S:X  a  [        S5      e[        R                  SU-  5        UR                  5       $ [        R                  " US9[        [        R                  " 5       5      S.n[         S	:X  aY  ["        R$                  " S
5      =(       d    ['        5       R)                  5       nU(       a  [        R+                  SU  35        UUS'   ['        5       nUR-                  XS9nUc  [        R.                  " 5       nU[         :X  Ga  U	(       a   ["        R0                  R3                  U	5      nOiU(       a1  ["        R0                  R4                  " U/U R7                  S5      Q76 nO1["        R0                  R4                  " US/U R7                  S5      Q76 n[9        SU SU 35        UR;                  XUUS9nUS   nS["        R<                  ;   a	  0 UESS0EnO0 UESS0EnUR>                  b  UR>                  US'   URA                  U USUc  SOUUUS9n[C        UUUU US S UUUUUUUU
UUUS9  SU ;   a  [E        UU 5      n["        R0                  RG                  U5      (       a  [        R+                  S5        GOp[        R+                  SU S 35         ["        RH                  " ["        R0                  R3                  U5      USS!9  GO!GOU[L        :X  Ga  U	(       a   ["        R0                  R3                  U	5      nOiU(       a1  ["        R0                  R4                  " U/U R7                  S5      Q76 nO1["        R0                  R4                  " US$/U R7                  S5      Q76 n[9        S%U 35        [O        U 5      u  nnU=(       d    [P        n[        R+                  S&5        [S        UU UU5      nUc  [        RU                  S'U  35        g [C        UUUU UUUUUUUUUUU
UUUS9  URW                  WS(9  UR                  5       nU$ ! [J         a<    [        R                  S"U S#["        R0                  R3                  U5       S35         Nff = f))Nr4   r5   )r*   r$   r/   r   zCannot find the requested files in the cached path and outgoing traffic has been disabled. To enable look-ups and downloads online, set 'local_files_only' to False.z6We can not confirm the cached file is for revision: %s)r%   )z
user-agentzsnapshot-identifierr7   INTRA_CLOUD_ACCELERATION_REGIONz6Intra-cloud acceleration enabled for downloading from zx-aliyun-region-id)r.   r/   r;   modelszDownloading Model from z to directory: )r#   r'   endpointRevisionCI_TESTzsnapshot-ci-testTrueSnapshotcached_model_revisionTF)r"   r#   	recursiveuse_cookiesheadersrR   )
r/   r#   r'   r(   r)   r,   r+   r-   rR   r1   .z3Target directory already exists, skipping creation.zCreating symbolic link [z].)target_is_directoryzFailed to create symbolic link z for datasetsz"Downloading Dataset to directory: z"Fetching dataset repo file list...z*Failed to retrieve file list for dataset: )revision_info),r   r   r   r   r   lencached_filesr=   loggerwarningget_root_locationr   get_user_agentstruuiduuid4r   r>   getenvr   !_get_internal_acceleration_domaininfoget_endpoint_for_readget_cookiesrB   abspathrC   splitprintget_valid_revision_detailr?   rW   get_model_files_download_file_listsr   existssymlinkOSErrorr   r   r   fetch_repo_fileserrorsave_model_version)r.   r/   r#   r$   r%   r&   r'   r(   r)   r*   r+   r,   r-   r1   temporary_cache_dircacherF   rZ   	region_id_apirR   	directoryrevision_detailsnapshot_header
repo_filesmasked_directorygroup_or_ownernamecache_root_paths                                rI   rE   rE      s   " #	)) )+< > ? 	? "F	"P ) 59;S <Lu!!"a'<= = 	O!" 	#&& 
 	
 ++zD#&tzz|#4
 $v-		;< @8==?  LWIV 1:,-x-- . 2?&224G'GGOOI6	GGLLKc8JK	GGLLx >*1--*<>	)(?9+NP"<<Gh = PO&z2H BJJ&"MW"M1CV0L"M"EW"EV0D"E**6/4/J/J  +- -- !%,_E''! . #J !##!$7#5 /-'!#5%( g~#=w$( 77>>),,KKMO KK":9+R HI

GGOO,<=%046 $ ++GGOOI6	GGLLKc8JK	GGLLz >*1--*<>	6ykBC#?#H ND&B*BOKK<=)$*24J !@	JL ##!$7#5 /-'!#5%* 	   ?113c # =i[bggoo^nNoMppqrs   3Q8 8AR>=R>c           
          SnSn/ n  U R                  UUSSUUUS9nUR	                  U5        [        U5      U:  a   U$ US-  nM@  ! [         a#  n[        R                  SU 35         S nAU$ S nAff = f)Nr      Tr;   )r.   r#   	root_pathrX   page_number	page_sizerR   zError fetching dataset files: )get_dataset_files	Exceptionra   rw   extendr_   )	r|   r.   r#   rR   r   r   r   dataset_fileses	            rI   rv   rv     s    KIJ
	 22!'#! 3 #M 	-(}	)  	q'   	LL9!=> 	s   A 
A5A00A5patternc                 R     [         R                  " U 5        g! [         a     gf = f)NTF)recompileBaseException)r   s    rI   _is_valid_regexr     s(    


7 s    
&&patternsc                     [        U [        5      (       a  U /n U b,  U  Vs/ s H  oR                  S5      (       d  UOUS-   PM!     n nU $ s  snf )Nr;   *)
isinstancere   endswith)r   items     rI   _normalize_patternsr     sY    (C  :GO
GOtc**Ds
:x 	 
 O
s   &A
c                 j    U b0  / nU  H&  n[        U5      (       d  M  UR                  U5        M(     U$ g )N)r   append)r   regex_patternsr   s      rI   _get_valid_regex_patternr     s;    Dt$$%%d+      r   rz   ry   apir   r   rR   c                   ^^^^^^^^^	^
^^^ [        U5      n[        U5      n[        U5      n[        U5      n[        U5      n/ nU  GH  mTS   S:X  a  M   U(       a;  [        U Vs/ s H  n[        R                  " TS   U5      PM     sn5      (       a  MR  U(       a;  [        U Vs/ s H  n[        R                  " TS   U5      PM     sn5      (       a  M  U(       a=  [        U Vs/ s H  n[        R
                  " UTS   5      S LPM!     sn5      (       a  M  Ub#  U(       a  [        U4S jU 5       5      (       d  M  Ub$  U(       a  [        U4S jU 5       5      (       d  GM%  TR                  T5      (       a>  [        R                  R                  TS   5      n[        R                  SU S35        GMy   UR                  T5        GM     [        US
S9UUU
UUUUUUUU	U4S j5       n[!        U5      S:  aG  [        R#                  S[!        U5       S35        U" U5        [        R#                  ST ST S35        g g s  snf s  snf s  snf ! [         a$  n[        R                  S	U-  5         S nAGM=  S nAff = f)Nr	   treer   Namec              3   X   >#    U  H  n[         R                   " TS    U5      v   M!     g7fr   Nfnmatch.0r   	repo_files     rI   	<genexpr>'_download_file_lists.<locals>.<genexpr>  s+      7'5G  	&(97CC'5   '*c              3   X   >#    U  H  n[         R                   " TS    U5      v   M!     g7fr   r   r   s     rI   r   r     s+      ;'9G  	&(97CC'9r   zFile z8 already in cache with identical hash, skip downloading!z The file pattern is invalid : %sF)r-   disable_tqdmc                    > T[         :X  a  [        T
U S   TTS9nO6T[        :X  a  TR                  U S   TTTTS9nO[	        ST S[
         35      e[        UU TTTTST	S9  g )Nr   )r"   	file_pathr#   rR   )	file_namedataset_name	namespacer#   rR   r4   z, supported types: F)r   r1   )r   r   r   get_dataset_file_urlr   r   r   )r   urlr   rz   r'   rR   r   rZ   r   r1   r.   r/   r#   ry   s     rI   _download_single_file3_download_file_lists.<locals>._download_single_file+  s    '' #F+!!	#C
 ++**#F+!(!! + #C #%i[0CDUCVW  	1		
r   r   zGot z files, start to download ...z	Download z 'z' successfully.)r   r   anyr   r   searchrs   r>   rB   basenamera   debugr   r   rb   r   r_   rj   )r   rz   ry   r.   r   r   r   rZ   r/   r#   r'   r(   r)   r+   r,   r-   rR   r1   ignore_regex_patternfiltered_repo_filesr   r   r   r   r   s    ``````````     ``      @rI   rr   rr     s   ( */:O(8N-.AB,-?@34GH	V&)	23#2(#2 OOIf$5w?#2( $ $ "s#6,#6 OOIf$5w?#6, ( ( ##7-#7 IIgy'89E#7- ) ) )n 7'57 7 7 !-2D ;'9; ; ; ||I&&GG,,Yv->?	I;&^_  '  &&y1Y  \ 5A
 
 B
< !#3*+,,IJ	L12i	{"WI_EF	 $Q(,-0  	CNN=ABB	Csa   I)$I
II+$I
II-&I
I!$I$I.AII
J'JJ)=r   r>   r   rf   
contextlibr   http.cookiejarr   pathlibr   typingr   r   r   r	   r
   modelscope.utils.constantr   r   r   r   r   r   modelscope.utils.file_utilsr   modelscope.utils.loggerr   modelscope.utils.thread_utilsr   r   r   r   callbackr   	constantsr   errorsr   file_downloadr   r   r   utils.cachingr   utils.utilsr   r   r    r!   ra   re   boolintrJ   rN   rE   rv   r   r   r   rr    r   rI   <module>r      s    	 	  " $  4 4: :
 A . 9 ) & * $B B /* * 
 "(,-1',#';?:>#6:7;!%.'+7;!s3s3sms3 S$_%s3 tSy)*	s3
 tns3 i s3 "%T#Y"78s3 !sDI~!67s3 }s3 U49c>23s3 eDIsN34s3 #s3 s3 }s3 tns3  T"234!s3" 	#s3p 7(,#-1',#';?:>6:7;'+\%\%sm\% S$_%\% }	\%
 tSy)*\% tn\% i \% "%T#Y"78\% !sDI~!67\% U49c>23\% eDIsN34\% tn\% \% 	\%D  $4(,-1',#';?:>#6:7;7;mm }m sm	m
 S$_%m tSy)*m tnm i m "%T#Y"78m !sDI~!67m }m U49c>23m eDIsN34m m T"234m`:S %T#Y"7 tCy (  $4#';?:>6:7;"7;%mGS	mGmG mG 	mG
 
mG mG mG }mG smmG i mG "%T#Y"78mG !sDI~!67mG U49c>23mG eDIsN34mG  !mG" sm#mG$ T"234%mGr   