
    9isu                     .   S SK r S SKrS SKrS SKrS SKrS SKrS SKrS SKrS SKJ	r	  S SK
Jr  S SKJr  S SKJr  S SKJrJrJrJrJr  S SKrS SKJr  S SKJr  S S	KJrJr  S S
KJrJ r J!r!J"r"J#r#J$r$J%r%  S SK&J'r'J(r(J)r)J*r*J+r+J,r,  S SK-J.r.J/r/  S SK0J1r1  SSK2J3r3J4r4  SSK5J6r6J7r7J8r8  SSK9J:r:  SSK;J<r<J=r=J>r>  \1" 5       r?\(SSSSS4S\@S\@S\\@   S\\@   S\\\@S4   S\\A   S\\   S\\@   S\\@   4S jjrB\'SSSSS4S\@S\@S\\@   S\\@\S4   S\\@   S\\\\@4      S\\A   S\\   S\@4S jjrCS\(SSSSSSS .S!\@S\@S"\@S\\@   S\\@   S\\\@S4   S\\A   S\\   S\\@   S#\AS\\@   4S$ jjrDS\@S\@4S% jrESS\+4S\@S\@S\@S"\@4S& jjrF S3S\@S\@S\@S'\\@   4S( jjrGS) rH     S4S*\@S\@S+\@S\S,\\\@\@4      S-\IS#\AS.\\\3      S'\@4S/ jjrJ   S5S*\@S\@S+\@S-\IS\S,\\\@\@4      S#\AS.\\\3      4S0 jjrK S3S*\@S\@S+\@S\S,\\\@\@4      4
S1 jjrL  S6S.\\\3      4S2 jjrMg)7    N)ThreadPoolExecutor)partial)	CookieJar)Path)DictListOptionalTypeUnion)Retry)tqdm)HubApiModelScopeConfig)API_FILE_DOWNLOAD_CHUNK_SIZEAPI_FILE_DOWNLOAD_RETRY_TIMESAPI_FILE_DOWNLOAD_TIMEOUT	FILE_HASHMODELSCOPE_DOWNLOAD_PARALLELS)MODELSCOPE_PARALLEL_DOWNLOAD_THRESHOLD_MBTEMPORARY_FOLDER_NAME)DEFAULT_DATASET_REVISIONDEFAULT_MODEL_REVISIONINTRA_CLOUD_ACCELERATIONREPO_TYPE_DATASETREPO_TYPE_MODELREPO_TYPE_SUPPORT)get_dataset_cache_rootget_model_cache_root)
get_logger   )ProgressCallbackTqdmCallback)FileDownloadErrorInvalidParameterNotExistError)ModelFileSystemCache)file_integrity_validationget_endpointmodel_id_to_group_owner_nameFmodel_id	file_pathrevision	cache_dir
user_agentlocal_files_onlycookies	local_dirreturnc                 ,    [        U U[        UUUUUUS9	$ )a  Download from a given URL and cache it if it's not already present in the local cache.

Given a URL, this function looks for the corresponding file in the local
cache. If it's not there, download it. Then return the path to the cached
file.

Args:
    model_id (str): The model to whom the file to be downloaded belongs.
    file_path(str): Path of the file to be downloaded, relative to the root of model repo.
    revision(str, optional): revision of the model file to be downloaded.
        Can be any of a branch, tag or commit hash.
    cache_dir (str, Path, optional): Path to the folder where cached files are stored.
    user_agent (dict, str, optional): The user-agent info in the form of a dictionary or a string.
    local_files_only (bool, optional):  If `True`, avoid downloading the file and return the path to the
        local cached file if it exists. if `False`, download the file anyway even it exists.
    cookies (CookieJar, optional): The cookie of download request.
    local_dir (str, optional): Specific local directory path to which the file will be downloaded.

Returns:
    string: string of local file or if networking is off, last version of
    file cached on disk.

Raises:
    NotExistError: The file is not exist.
    ValueError: The request parameter error.

Note:
    Raises the following errors:

        - [`EnvironmentError`](https://docs.python.org/3/library/exceptions.html#EnvironmentError)
        if `use_auth_token=True` and the token cannot be found.
        - [`OSError`](https://docs.python.org/3/library/exceptions.html#OSError)
        if ETag cannot be determined.
        - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)
        if some parameter value is invalid
	repo_typer,   r-   r.   r/   r0   r1   )_repo_file_downloadr   )r*   r+   r,   r-   r.   r/   r0   r1   s           \/var/www/html/land-doc-ocr/venv/lib/python3.13/site-packages/modelscope/hub/file_download.pymodel_file_downloadr8   +   s-    \ !)	 	    
dataset_idc                 ,    [        U U[        UUUUUUS9	$ )ag  Download raw files of a dataset.
Downloads all files at the specified revision. This
is useful when you want all files from a dataset, because you don't know which
ones you will need a priori. All files are nested inside a folder in order
to keep their actual filename relative to that folder.

An alternative would be to just clone a dataset but this would require that the
user always has git and git-lfs installed, and properly configured.

Args:
    dataset_id (str): A user or an organization name and a dataset name separated by a `/`.
    file_path (str): The relative path of the file to download.
    revision (str, optional): An optional Git revision id which can be a branch name, a tag, or a
        commit hash. NOTE: currently only branch and tag name is supported
    cache_dir (str, Path, optional): Path to the folder where cached files are stored, dataset file will
        be save as cache_dir/dataset_id/THE_DATASET_FILES.
    local_dir (str, optional): Specific local directory path to which the file will be downloaded.
    user_agent (str, dict, optional): The user-agent info in the form of a dictionary or a string.
    local_files_only (bool, optional): If `True`, avoid downloading the file and return the path to the
        local cached file if it exists.
    cookies (CookieJar, optional): The cookie of the request, default None.
Raises:
    ValueError: the value details.

Returns:
    str: Local folder path (string) of repo snapshot

Note:
    Raises the following errors:
    - [`EnvironmentError`](https://docs.python.org/3/library/exceptions.html#EnvironmentError)
    if `use_auth_token=True` and the token cannot be found.
    - [`OSError`](https://docs.python.org/3/library/exceptions.html#OSError) if
    ETag cannot be determined.
    - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)
    if some parameter value is invalid
r4   )r6   r   )r:   r+   r,   r-   r1   r.   r/   r0   s           r7   dataset_file_downloadr<   e   s-    \ #)	 	r9   )r5   r,   r-   r.   r/   r0   r1   disable_tqdmrepo_idr5   r=   c          
      |   U(       d  [         nU[        ;  a  [        SU< S[        < 35      e[        XXBS9u  pU(       a6  UR	                  U5      nUb  [
        R                  S5        U$ [        S5      e[        5       n[        R                  " US9[        [        R                  " 5       5      S.n[        S:X  aP  [        R                   " S	5      =(       d    UR#                  5       nU(       a  [
        R%                  S
U  35        XS'   Uc  [        R&                  " 5       n/ nUR)                  XS9nS nU[         :X  a  UR+                  XUUS9nUR-                  U USUc  SOUUS9nU Hb  nUS   S:X  a  M  US   U:X  d  M  UR/                  U5      (       a1  US   n[
        R1                  SU S35        UR3                  U5      s  $ Un  O   OU[4        :X  a  [7        U 5      u  nnU(       d  [8        nSnSn  UR;                  U USSUUUS9nSnU Hd  nUS   S:X  a  M  US   U:X  d  M  UR/                  U5      (       a1  US   n[
        R1                  SU S35        UR3                  U5      s  $ UnSn  O   [A        U5      U:  d  U(       a  OUS-  nM  Uc  [C        SU< SU < 35      eU[         :X  a  [E        XUU5      nO/U[4        :X  a  URG                  US   WWUUS9nO[        S U 35      e[I        UUXX5      $ ! [<         a&  n[
        R?                  SU  SU 35         S nAM  S nAff = f)!NzInvalid repo type: z, only support: )r1   r-   r5   z>File exists in local cache, but we're not sure it's up to datezCannot find the requested files in the cached path and outgoing traffic has been disabled. To enable look-ups and downloads online, set 'local_files_only' to False.)r.   )z
user-agentzsnapshot-identifiertrueINTRA_CLOUD_ACCELERATION_REGIONz6Intra-cloud acceleration enabled for downloading from zx-aliyun-region-id)r>   r5   )r,   r0   endpointTF)r*   r,   	recursiveuse_cookiesrB   r
   treer   NameFile z8 already in cache with identical hash, skip downloading!r    d   /)r>   r,   	root_pathrC   page_number	page_sizerB   zGet dataset: z file list failed, error: zThe file path: z not exist in: )	file_namedataset_name	namespacer,   rB   zInvalid repo type )%r   r   r$   $create_temporary_directory_and_cacheget_file_by_pathloggerwarning
ValueErrorr   r   get_user_agentstruuiduuid4r   osgetenv!_get_internal_acceleration_domaininfoget_cookiesget_endpoint_for_readget_valid_revisionget_model_filesexistsdebugget_file_by_infor   r)   r   get_dataset_files	Exceptionerrorlenr%   get_file_download_urlget_dataset_file_urldownload_file)r>   r+   r5   r,   r-   r.   r/   r0   r1   r=   temporary_cache_dircachecached_file_path_apiheaders	region_id
repo_filesrB   file_to_download_meta	repo_filerM   group_or_ownernamerK   rL   dataset_fileseis_existurl_to_downloads                                r7   r6   r6      s    #	)) )+< > ? 	? "F	"P
  11)<'NNP $#<= =
 8D '55N"4::<0G
  6)II78 8557 	 KKH	R -6()"..0J))')OH O#**( + L ))!(g * 
 $I F* I-<<	** )& 1ILL	{*bc !11)<<,5) $ 
'	';GD/H	 $ 6 6#%!" +'% !7 !' H*	V$.V$	1||I..$-f$5	#I;.fg  %55i@@09-#' + =!I-1KC F $&1 2 	2 O#/H08:	'	'33+F3$ 4  -i[9::*?,WG GW  #G9,FqcJLs   L 
L;L66L;c                 ~   U R                  [        R                  R                  5      (       a)  U R	                  [        R                  R                  5      n [        R                  R                  U 5      n[        R                  R                  U 5      nUS:X  a  g UR                  S5      (       d   [        R                  R                  US5      n[        U5      u  pEUR                  SS5      n[        R                  R                  XU5      n[        R                  R                  X$U5      n[        R                  R                  U5      (       aX  [        R                  R                  U5      (       d3  [        R                  SU SU 35         [        R                  " Xv5        g g g ! [         a     g f = f)Ndatasetshub.___zLegacy cache dir exists: z
, move to )endswithrY   pathsepstripdirnamebasenamejoinr)   replacera   rR   r\   shutilmovere   )r-   r*   legacy_cache_root	base_namert   ru   rk   legacy_cache_dirs           r7   !move_legacy_cache_to_standard_dirr   3  sQ   "''++&&OOBGGKK0		2  +IJ%%e,, GGLL):EB7AN<<U#D'',,y$Gww||$5tL	ww~~ &(ggnn5H&I&I'(8'9DWCXY	
	KK(> 'J  		s   F/ /
F<;F<c                    U[         :X  a  [        5       nO#U[        :X  a  [        5       nO[	        SU 35      e[        U 5      u  pVUb0  [        R                  R                  U[        5      n[        U5      nOsUc  Un[        X 5        [        U[        5      (       a  [        U5      n[        R                  R                  U[        XV5      nUR                  SS5      n[        X%U5      n[        R                   " USS9  Xx4$ )Nz7repo_type only support model and dataset, but now is : r}   r~   Texist_ok)r   r   r   r   rT   r)   rY   r   r   r   r&   r   
isinstancer   rV   r   makedirs)	r*   r1   r-   r5   default_cache_rootrt   ru   rk   rl   s	            r7   rP   rP   V  s     O#13	'	'35Ei[Q
 	
 8AN ggll96KL$Y/*I-iBi&&II ggll96K+9A||C'$YEKK#d3%%r9   rB   c                     [         R                  R                  U5      n[         R                  R                  U5      nSnU(       d
  [        5       nUR	                  UU UUS9$ )a  Format file download url according to `model_id`, `revision` and `file_path`.
e.g., Given `model_id=john/bert`, `revision=master`, `file_path=README.md`,
the resulted download url is: https://modelscope.cn/api/v1/models/john/bert/repo?Revision=master&FilePath=README.md

Args:
    model_id (str): The model_id.
    file_path (str): File path
    revision (str): File revision.
    endpoint (str): The remote endpoint

Returns:
    str: The file url.
zQ{endpoint}/api/v1/models/{model_id}/repo?Revision={revision}&FilePath={file_path})rB   r*   r,   r+   )urllibparse
quote_plusr(   format)r*   r+   r,   rB   download_url_templates        r7   rh   rh   v  s`    " ''	2I||&&x0Ho> ''	 (  r9   c           	         U u  pp4pVpxUc  0 O[         R                  " U5      n	[        [        R                  " 5       R
                  5      U	S'   [        [        SS/S9n
USU< SU< 3-   n  Sn[        R                  R                  U5      (       aO  [        US5       nUR                  S[        R                  5      nU H  nUR                  U5        M     S S S 5        X<-   nX:  a  g S	U< S
U< 3U	S'   [        US5       n[         R"                  " USU	U[$        S9nUR'                  [(        S9 H@  nU(       d  M  UR+                  U5        U H  nUR                  [-        U5      5        M     MB     S S S 5        g ! , (       d  f       N= f! , (       d  f       N = f! [.         aI  nU
R1                  SUUS9n
[2        R5                  SU< SU< S35        U
R7                  5          S nAOS nAff = fGM  )NX-Request-IDr    GETtotalbackoff_factorallowed_methods_Tr   rbbytes=-Rangeab+streamro   r0   timeout
chunk_sizerf   zDownloading: z failed, reason: z will retry)copydeepcopyrV   rW   rX   hexr   r   rY   r   ra   openseekioSEEK_ENDupdaterequestsgetr   iter_contentr   writerg   re   	incrementrR   rS   sleep)paramsmodel_file_pathprogress_callbacksstartendurlrM   r0   ro   get_headersretrypart_file_namepartial_lengthfcallbackdownload_startrchunkrw   s                      r7   download_part_with_retryr     s   X^UOSW"T]]7-CK"%djjl&6&6"7K+!E %5#'>>N
	Nww~~"$ $.$/1%&VVAr{{%;N$6 7 %7 0 #3N# $4BC#HK ne,LL'#57 ^^#? , AEu(:H$OOCJ7 );	A - + 0/ -,  	OOE3aO8ENN+Q0 1KKMM		7 sT   -2F$ ;FF$ ,F$ 8F8F9F$ 
FF$ 
F!F$ $
G7.?G22G7r   rM   ro   	file_sizer   c	                    Uc  / OUR                  5       nU(       d  UR                  [        5        U V	s/ s H
  o" X%5      PM     nn	Sn
/ n[        R                  R                  X5      n[        R                  " [        R                  R                  U5      SS9  [        [        XZ-  5      5       H(  nX-  nUS-   U
-  S-
  nUR                  XXU X#U45        M*     WS-   U:  a  UR                  XUS-   US-
  XX445        [        [        S5      n[        USS9 n[        UR                  [        U5      5        S S S 5        U H  n	U	R!                  5         M     ["        R$                  " 5       n['        [        R                  R                  X5      S5       nU H  nUS	   S
US   < S
US   < 3-   n['        US5       n UR)                  S[*        -  5      nU(       d  O$UR-                  U5        UR/                  U5        MD  S S S 5        [        R0                  " U5        M     S S S 5        UR3                  5       $ s  sn	f ! , (       d  f       GN= f! , (       d  f       NW= f! , (       d  f       UR3                  5       $ = f)Ni   
Tr   r       download)max_workersthread_name_prefixwbr   r         r   )r   appendr"   rY   r   r   r   r   rangeintminr   r   listmapr   r   hashlibsha256r   readr   r   r   remove	hexdigest)r   r1   rM   r0   ro   r   r=   r   rB   r   	PART_SIZEtasksr+   idxr   r   	parallelsexecutorhash_sha256output_filetaskr   	part_filer   s                           r7   parallel_downloadr     sK     29?Q?V?V @!!,/7I7I8&7I   "IEY2IKK	*T:S./0Qw)#a'iU'3 	4 1
 QwiS1Wi!mg8 	9126I	!j
BEMX\\2E:;
B ' ' .."K	bggll90$	7;D!!W47DG'DDNnd+y%NN20L+LME %%e,&&u-  , IIn%  
8   ""I$
B 
B ,+ 
8	7   ""s6   I I(I.AI"I.
I
I+'I..
Jc           	      ~   Uc  / OUR                  5       nU(       d  UR                  [        5        U Vs/ s H
  o" X#5      PM     nnUc  0 O[         R                  " U5      n	[	        [
        R                  " 5       R                  5      U	S'   [        R                  R                  X5      n
[        R                  " [        R                  R                  U
5      SS9  [        R                  SX
5        Sn[        R                   " 5       n[#        [$        SS/S	9n  US
:X  a1  ['        U
S5         U H  nUR)                  S5        M     SSS5        GO@S
n[        R                  R+                  U
5      (       aQ  Sn['        U
S5       nUR-                  S
[.        R0                  5      nU H  nUR)                  U5        M     SSS5        X:  a  OSU< SUS-
  < 3U	S'   ['        U
S5       n[2        R4                  " U SU	U[6        S9nUR9                  5         UR;                  [<        S9 HZ  nU(       d  M  U H  nUR)                  [?        U5      5        M     URA                  U5        U(       a  MI  UR)                  U5        M\     SSS5         U H  nURI                  5         M     U(       a  S$ URK                  5       $ s  snf ! , (       d  f       GN= f! , (       d  f       GN!= f! , (       d  f       Nj= f! [B         a-  nSnURE                  SU US9nURG                  5          SnAOSnAff = fGM!  )a  Download remote file, will retry 5 times before giving up on errors.

Args:
    url(str):
        actual download url of the file
    local_dir(str):
        local directory where the downloaded file stores
    file_name(str):
        name of the file stored in `local_dir`
    file_size(int):
        The file size.
    cookies(CookieJar):
        cookies used to authentication the user, which is used for downloading private repos
    headers(Dict[str, str], optional):
        http headers to carry necessary info when requesting the remote file
    disable_tqdm(bool, optional): Disable the progress bar with tqdm.
    progress_callbacks(List[Type[ProgressCallback]], optional):
        progress callbacks to track the download progress.

Raises:
    FileDownloadError: File download failed.

Nr   Tr   downloading %s to %sFr    r   r   r   zw+r   r   r   r   r   r   r   r   )&r   r   r"   r   rV   rW   rX   r   rY   r   r   r   r   rR   rb   r   r   r   r   r   r   ra   r   r   r   r   r   r   raise_for_statusr   r   rg   r   re   r   r   r   r   )r   r1   rM   r   r0   ro   r=   r   r   r   temp_file_path	has_retryr   r   r   r   r   r   rw   s                      r7   http_get_model_filer     s   B  29?Q?V?V @!!,/7I7I8&7I    "T]]7-CK"%djjl&6&6"7KWW\\)7NKK/$?
LL'=I.."K+!E
 -	A~.$/$6 * %7 0 Nww~~n-- 	.$/1%&VVAr{{%;N$6 7 %7 0 * + 5C4=M$CK ne,LL'#57 ""$^^#? , AEu(:H$OOCJ7 );(y'..u5A -" 
 ' ' 49+"7"7"99E( 0/ 0/ -,$  	IOOE3aO8EKKMM	W s~   KL 1KL 4L 
;KL L /AK1;9K18K1L 
KL 
K.)L 1
K?;L 
L9#L44L9c           
      L   Sn[        [        R                  SUSS9nUc  0 O[        R                  " U5      n[        [        R                  " 5       R                  5      US'   U" 5        n[        R                  SXR                  5        [        [        SS	/S
9n	  UR                  5       n
SU
-  US'   [        R                   " U SUU["        S9nUR%                  5         UR&                  R!                  S5      nUb  [)        U5      OSn[+        SSSUU
SU-   S-   S9nUR-                  [.        S9 H7  nU(       d  M  UR1                  [3        U5      5        UR5                  U5        M9     UR7                  5          SSS5        [        R                  SX5        [>        R@                  RC                  WR                  5      nUU:w  aO  [>        RD                  " UR                  5        SU< SU< SU< S3n[        RG                  U5        [I        U5      e[>        RJ                  " UR                  [>        R@                  RM                  X5      5        g! [8         a*  nU	R;                  S	XS9n	U	R=                  5          SnAOSnAff = fGM  ! , (       d  f       GN= f)a2  Download remote file, will retry 5 times before giving up on errors.

Args:
    url(str):
        actual download url of the file
    local_dir(str):
        local directory where the downloaded file stores
    file_name(str):
        name of the file stored in `local_dir`
    cookies(CookieJar):
        cookies used to authentication the user, which is used for downloading private repos
    headers(Dict[str, str], optional):
        http headers to carry necessary info when requesting the remote file

Raises:
    FileDownloadError: File download failed.

r   F)modedirdeleteNr   r   r    r   r   Tz	bytes=%d-r   r   zContent-LengthBi   zDownloading [])unit
unit_scaleunit_divisorr   initialdescr   r   zstoring %s in cache at %srG   z& download incomplete, content_length: z5 but the                     file downloaded length: z, please download again)'r   tempfileNamedTemporaryFiler   r   rV   rW   rX   r   rR   rb   ru   r   r   tellr   r   r   r   ro   r   r   r   r   r   rg   r   closere   r   r   rY   r   getsizer   rf   r#   r   r   )r   r1   rM   r0   ro   r   temp_file_managerr   	temp_filer   downloaded_sizer   content_lengthprogressr   rw   downloaded_lengthmsgs                     r7   http_get_filer  ^  sD   2 E##$IeM"T]]7-CK"%djjl&6&6"7K			+S..A/"G% "+.."2'2_'DG$LL'#57 ""$!"/?!@'5'A "$GK #!%+(94s: ^^#? , AEu E
3!.	A
  G 
P LL,c=	7!!
		)..! u/1 	S$$JJy~~rww||IAB  s<;  
	s=   *2JBI??I>J
J$ J	J	JJ
J#c                    [         S-  S-  US   :  a4  [        S:  a*  [        U UUS   UUc  S OUR                  5       US   UUS9nO[	        U UUS   US   UUUUS9n[
        R                  R                  X!S   5      n	[        U;   a3  U[           n
Ub  X:w  a  [        S5        [        X5        O[        X5        UR                  X5      $ )Ni  Sizer    r   )ro   r0   r   r=   r   )r   ro   r0   r=   r   zLMismatched real-time digest found, falling back to lump-sum hash computation)r   r   r   get_dictr   rY   r   r   r   printr'   put_file)r   	file_metark   rl   ro   r0   r=   r   file_digestr   expected_hashs              r7   rj   rj     s     147$>B 59'f#OD1A1A1C'%1	
 *f'%1	
 0F2CDII!),"+b *)C%i?>>)//r9   )N)NNFNN)NFN)FN)Nr   r   r   rY   r   r   r   rW   concurrent.futuresr   	functoolsr   http.cookiejarr   pathlibr   typingr   r   r	   r
   r   r   requests.adaptersr   	tqdm.autor   modelscope.hub.apir   r   modelscope.hub.constantsr   r   r   r   r   r   r   modelscope.utils.constantr   r   r   r   r   r   modelscope.utils.file_utilsr   r   modelscope.utils.loggerr   r   r!   r"   errorsr#   r$   r%   utils.cachingr&   utils.utilsr'   r(   r)   rR   rV   boolr8   r<   r6   r   rP   rh   r   r   r   r   r  rj    r9   r7   <module>r      sc     	 	     1  $  4 4  #  7F F F: :
? . 4 F F /8 8 
 5#)-',#'#777 sm7 }	7
 dCo&7 tn7 i 7 }7 c]7z 7(,#-1',#'777 sm7 S$_%	7
 }7 tSy)*7 tn7 i 7 	7| 4#)-',#'#QGQGQG 	QG
 smQG }QG dCo&QG tnQG i QG }QG QG c]QGh     H ;?:>:I&3 &47&47& 58&F 59C %($' %-SM<)` ;?'++0IM&*1#3 1#!$1#!$1#  )1#  (S#X7	1#
 "%1# %)1# +/t4D/E*F1# !$1#t )-7;g:	g:g: g: 	g:
 g: d38n%g: g: T"234g:^ )-OC	OCOC OC 	OC
 d38n%OCr 7;10 T"23410r9   