
    9i;                     ^   S r SSKrSSKrSSKrSSKrSSKrSSKJr  SSKJ	r	J
r
JrJr  SSKJr  SSKJrJrJr  SSKJrJr  SSKJrJrJrJrJrJrJr  SS	KJrJ r   SS
K!J"r"  SSK#J$r$  SSK%J&r&  SSK'J(r(  SSK!J)r)  SSK*JrJ+r+  SSK,J-r-  S r.SSSSS\S\S4	S\\/   S\\0   S\\/   S\\\
\/   \/4      S\\\
\/   \/4      S\1S\/S\\/   S\/S\/4S jjr2S\SSSSS\SS.	S\/S\\/   S\\/   S \\/\(S4   S\\0   S\\/   S\\\
\/   \/4      S\\\
\/   \/4      S\1S\/4S! jjr3S" r4S\SS\S4S#\
\5   S$\$S%\/S\/S\\/   S\\/   S\\\
\/   \/4      S\\\
\/   \/4      S\1S\/4S& jjr6S' r7S(\/4S) jr8S*\\/\
\/   4   4S+ jr9S*\
\/   4S, jr:\S4S-\/S\\/   S\/S\
\5   \544S. jjr;SS/.S\/S0\/S\\/   4S1 jjr<g)2u8   
本文件实现了星河社区git仓库的下载功能
    N)Class)DictListOptionalUnion)log)$create_temporary_directory_and_cachedownload_fileget_git_info)InvalidParameterraise_on_error)REPO_TYPE_SUPPORTREPO_TYPE_DATASETREPO_TYPE_MODELSTUDIO_GIT_HOST_DEFAULTDEFAULT_MODEL_REVISIONDEFAULT_DATASET_REVISIONDEFAULT_MAX_WORKERS)thread_executorget_model_masked_directory)quote)ModelFileSystemCache)_header_fill)Path)urlparse   )r   NotExistError)post_repo_statisticc                     [         R                  " SSS9(       d  [         R                  " S[         R                  " S5      S9n U  S3n[         R                  R                  U5      (       aE  [	        US5       nUR                  5       R                  5       [         R                  S'   SSS5        ggg! , (       d  f       g= f)	uv   初始化函数，从本地磁盘加载AI Studio认证令牌。

Args:
    无参数。

Returns:
    无返回值。
AISTUDIO_ACCESS_TOKEN defaultAISTUDIO_CACHE_HOMEHOMEz/.cache/aistudio/.auth/tokenrN)osgetenvpathexistsopenreadstripenviron)
cache_hometoken_file_pathfiles      ^/var/www/html/land-doc-ocr/venv/lib/python3.13/site-packages/aistudio_sdk/snapshot_download.pyinitr3   #   s     99,b9YY4bii>OP
'L(DE77>>/**os+t6:iik6G6G6I

23 ,+ + : ,+s   90B44
CFrevisionlocal_files_only	local_dirallow_patternsignore_patternsmax_workersrepo_id	repo_typetokenreturnc	                 N   [        5         U(       d  [        S5      eU[        ;  a  [        SU S[         35      eU c  U[        :X  a  [        O[
        n SU0n	 [        X`U	5        [        UUU UUUUUUS9	$ ! [         a#  n
[        R                  " SU
 35         Sn
A
N9Sn
A
ff = f)preparez*Please provide a valid model_id or repo_idInvalid repo type: , only support: Nrepozrequest.dot.fail: )r;   r4   r5   r6   r8   r7   r9   r<   )r3   
ValueErrorr   r   r   r   r   	Exceptionr   debug_snapshot_download)r4   r5   r6   r7   r8   r9   r:   r;   r<   actiones              r2   snapshot_downloadrI   7   s     	FEFF))!),<=N<OP
 	
 /8<M/M+SigF,Gv6 )'%
 
  ,		&qc*++,s   A7 7
B$BB$)	r;   r4   	cache_dirr5   r6   r7   r8   r9   r<   rJ   c       	            U(       d  [         nU[        ;  a  [        SU< S[        < 35      e[        XUS9u  p[        R
                  R                  [        R                  " S[        5       5      5      nU(       aM  [        UR                  5      S:X  a  [        S5      e[        R                  " SU-  5        UR                  5       $ S[        [         R"                  " 5       5      0nU[         :X  d  U[$        :X  Gan  U(       a   [        R
                  R'                  U5      nOtU(       a1  [        R
                  R(                  " U/U R+                  S	5      Q76 nO<[        R
                  R(                  " X[         :X  a  S
OS/U R+                  S	5      Q76 n[        R,                  " SU 35        [/        U UU	S9u  nn[1        UUU
U UUUUUU	S9
  SU ;   a  [3        X5      n[        R
                  R5                  U5      (       a  [        R,                  " S5        ON[        R,                  " SU S35         [        R6                  " [        R
                  R'                  U5      USS9  UR;                  WS9  UR                  5       nU$ ! [8         a=    [        R                  " SU S[        R
                  R'                  U5       S35         Ngf = f)z
download repo
r@   rA   )r6   r;   AISTUDIO_CACHEr   zCannot find the requested files in the cached path and outgoing traffic has been disabled. To enable look-ups and downloads online, set 'local_files_only' to False.z6We can not confirm the cached file is for revision: %szsnapshot-identifier/modelsdatasetz,Downloading Model from remote to directory: )model_idr4   r<   )

repo_filescachetemporary_cache_dirr:   r;   r4   r8   r7   r9   r<   .z3Target directory already exists, skipping creation.zCreating symbolic link [z].T)target_is_directoryzFailed to create symbolic link z for )revision_info)r   r   r   r	   r'   r)   
expanduserr(   get_default_aistudio_cache_dirlencached_filesrC   r   warnget_root_locationstruuiduuid4r   abspathjoinsplitinfoget_model_files_download_file_listsr   r*   symlinkOSErrorsave_model_version)r:   r;   r4   rJ   r5   r6   r7   r8   r9   r<   rS   rR   system_cacheheaders	directoryrQ   revision_detailmasked_directorycache_root_paths                      r2   rF   rF   a   s     #	)) )+< > ? 	? "F	";77%%
		"$B$DEGLu!!"a'<= = 	I!" 	#&& 
 	

 "3tzz|#4
 '98I+IGGOOI6	GGLLKc8JK	GGLLP_C_xen >*1--*<>	HH>ykJL +: !+'J
 !%$7#! /-' g~#=$( 77>>),,HHMO HH7	{"EF

GGOO,<=%046 	   ?113 # =i[bggoo^nNoMppqrs   ;3J AKKc                      [         R                  R                  [        R                  " 5       R                  SSS5      5      n U $ )z'
default base dir: '~/.cache/aistudio'
z.cacheaistudiohub)r'   r)   rW   r   homejoinpath)default_cache_dirs    r2   rX   rX      s9     **499;+?+?*e,% &    rQ   rR   rS   c
           
        ^^^^^^	^ [        U5      n[        U5      n/ n
U  H  mTS   S:X  a  M   U(       a;  [        U Vs/ s H  n[        R                  " TS   U5      PM     sn5      (       a  MQ  Ub#  U(       a  [        U4S jU 5       5      (       d  Mw  U
R                  T5        M     [        USS9UUUUUU	4S	 j5       n[        U
5      S
:  aI  [
        R                  " S[        U
5       S35        U" U
5        [
        R                  " ST ST S35        ggs  snf ! [         a%  n[
        R                  " SU-  5         SnAGM,  SnAff = f)z
download all
typetreer)   Nc              3   X   >#    U  H  n[         R                   " TS    U5      v   M!     g7f)r)   N)fnmatch).0pattern	repo_files     r2   	<genexpr>'_download_file_lists.<locals>.<genexpr>   s+      7'5G  	&(97CC'5s   '*z The file pattern is invalid : %sF)r9   disable_tqdmc                 p  > [        TU S   T
T5      n TR                  U 5      (       a=  [        R                  R	                  U S   5      n[
        R                  " SU S35        gT	[        :X  a  U S   nTR                  S5      u  p4UR                  5       nUR                  5       n[        R                  " S[        S9nU S[        US	S
9 S[        US	S
9 S[        US	S
9 3nT
S:w  a  US[        T
S	S
9 3-  nOT	[        :X  a  U S   nTR                  S5      u  p4UR                  5       nUR                  5       n[        R                  " S[        S9nU S[        US	S
9 S[        US	S
9 S[        US	S
9 3nT
S:w  a  US[        T
S	S
9 3-  nO[        ST	 S[         35      e[!        UU TTSTS9  g)zdownload each filer)   z
File z8 already in cache with identical hash, skip downloading!NrM   STUDIO_GIT_HOSTr"   /api/v1/repos/r!   )safez/media/masterz?ref=r@   z, supported types: F)r   r<   )r   r*   r'   r)   basenamer   rc   r   rb   r-   r(   r   r   r   r   r   r
   )r}   	file_name	file_path	user_name	repo_namegit_hosturlrR   r:   r;   r4   rS   r<   s          r2   _download_single_file3_download_file_lists.<locals>._download_single_file   s    !)F*;XuM	<<	""((6):;IHH)$\] '!&)I#*==#5 I!)I!)Iyy!2<STH*N,-Q,-W,-/  8#uXB7899++!&)I#*==#5 I!)I!)Iyy!2<STH*N,-Q,-W,-/  8#uXB7899"%i[0CDUCVW  		
ru   r   zGot z files, start to download ...z	Download z 'z' successfully.)
_normalize_patternsanyrz   appendrD   r   r[   r   rY   rc   )rQ   rR   rS   r:   r;   r4   r7   r8   r9   r<   filtered_repo_filesr|   rH   r   r}   s    `````   `    @r2   re   re      sb    */:O(8N	V&	2 3#2(#2 OOIf$5w?#2( $ $  )n 7'57 7 7   &&y1;  @ 5A2
 2
 B2
h !#3*+,,IJ	L129YKr'/BC	 $_((  	=HH7!;<<	=s/   D$$D
%D$3$D$D$$
E.EEc           
         SnSn/ n U R                  UUUSSUUS9nSU;   a	  US   S:X  d#  [        R                  " SUS	    S
US    35        gUS   S   nUR                  U5        [	        U5      U:  a   U$ US-  nMy  )zget repo metar      TrM   )dataset_name	namespacer4   	root_path	recursivepage_number	page_sizeCode   z;Get dataset file list failed, request_id:                  	RequestIdz, message: MessageNDataFiles)list_repo_treer   errorextendrY   )	_apinamegroup_or_ownerr4   r   r   rQ   files_list_treecur_repo_filess	            r2   fetch_repo_filesr   7  s    KIJ
--$# . ! /)of.E.LII  -.k/):T9UW  (09.)~*  	q- ru   r|   c                 R     [         R                  " U 5        g! [         a     gf = f)checkTF)recompileBaseException)r|   s    r2   _is_valid_regexr   X  s(    


7 s    
&&patternsc                     [        U [        5      (       a  U /n U b,  U  Vs/ s H  oR                  S5      (       d  UOUS-   PM!     n nU $ s  snf )	normalizerM   *)
isinstancer]   endswith)r   items     r2   r   r   a  sY    (C  :GO
GOtc**Ds
:x 	 
 O
s   &A
c                 j    U b0  / nU  H&  n[        U5      (       d  M  UR                  U5        M(     U$ g)zprocess regexN)r   r   )r   regex_patternsr   s      r2   _get_valid_regex_patternr   l  s;    Dt$$%%d+  ru   rP   c           	      B   [         R                  " S[        S9nU SU  SU 3n[        US9n[	        [
        R                  " 5       R                  5      US'   SnSn/ nS	n	0 n
[        R                  " XES
9nUR                  S;   a  UR                  5       S   S   nU	(       a  U SU  SU SU SU 3	n[        R                  " XS
9nUR                  S;  a  [        U5        [        S5      eUR                  5       n[        U5        US:X  a;  [        US   5      n[         R                   R#                  UR                   5      nSU0n
UR                  S/ 5       H  nUS   S;   a  M  UR%                  U5        M!     UR                  SS5      n	US-  nU	(       a  M  X4$ )zList the models files.

    r   r"   r   z/tags/)r<   zX-Request-IDr   i  T)rj   )r      commitshaz/git/trees/z?recursive=true&page=z
&per_page=zrepo not foundr   Revisionrx   r)   )z
.gitignorez.gitattributes	truncatedF)r'   r(   r   r   r]   r^   r_   hexrequestsgetstatus_codejsonprintr   r   r   r)   r   r   )rP   r4   r<   endpointtag_pathrj   pageper_pagefilesr   rV   tag_resr)   r&   d
parsed_urlr   r1   s                     r2   rd   rd   x  s    yy*4KLH>(6(DH'G!$**,"2"23GNDHEIMll85Gj(<<>(+E2
>(;xjH]^b]ccmnvmwxLL/==
*!H 011FFHq19!!E(+J((9I'3MEE&"%DF|??LL &
 EE+u-		- ). ru   r4   filenamec                N    [        XS9nU Vs/ s H  oDS   PM	     nnX;   $ s  snf )a0  Get if the specified file exists

Args:
    repo_id (`str`): The repo id to use
    filename (`str`): The queried filename, if the file exists in a sub folder,
        please pass <sub-folder-name>/<file-name>
    revision (`Optional[str]`): The repo revision
Returns:
    The query result in bool value
r   r)   )rd   )r:   r   r4   r   r1   s        r2   file_existsr     s2      G7E&+,ed&\eE, -s   ")=__doc__r   r^   rz   r'   r   symtabler   typingr   r   r   r   aistudio_sdkr   aistudio_sdk.file_downloadr	   r
   r   aistudio_sdk.errorsr   r   aistudio_sdk.configr   r   r   r   r   r   r   aistudio_sdk.utils.utilr   r   urllib.parser   aistudio_sdk.utils.cachingr   aistudio_sdk.requests.hubr   pathlibr   r   errorsr   aistudio_sdk.dotr   r3   r]   boolintrI   rF   rX   dictre   r   r   r   r   rd   r    ru   r2   <module>r      sT      	 	  . . G G B6 6 6 R  ; 2  ! 3 0J* #',#6:7;*.'sm'tn' }' U49c>23	'
 eDIsN34' ' ' }' ' 	'Z  $4(,',#6:7;*[[ }[ sm	[
 S$_%[ tn[ }[ U49c>23[ eDIsN34[ [ [|  $46:7;*lDT
lDlD lD 	lD
 }lD smlD U49c>23lD eDIsN34lD lD lD^BS %T#Y"7 	tCy 	 /E, !, &sm,  ,  4j$	, f #'	 3-	ru   