
    9i                         S SK Jr  S SKrS SKrS SKJr  S SKJr  S SKJ	r	  S SK
Jr  S SKJrJrJr  S SKJr  \" 5       rS	rS
rSrSrSrSr " S S5      rg)    )print_functionN)hash_url_to_filename)HubApi)DataDownloadConfig)MS_CACHE_HOME)"DEFAULT_DATA_ACCELERATION_ENDPOINTMetaDataFields
UploadMode)
get_loggerAccessIdAccessSecretSecurityTokenBucket	BackupDirDirc            
       f    \ rS rSrS rS rS r\S 5       rS\	S\
4S jrS	\	S
\	S\S\S\	4
S jrSrg)OssUtilities   c                    U R                  US9  X l        X0l        X@l        [        R
                  R                  [        S5      U l        [        R                  " 5       U l        SU l        SU l        SU l        SS KnUR!                  U R                  S9U l        UR%                  U R                  S9U l        [)        5       U l        g )N)
oss_configztmp/resumable_storei   i      r   )root)_do_initdataset_name	namespacerevisionospathjoinr   resumable_store_root_pathmultiprocessing	cpu_countnum_threads	part_sizemultipart_thresholdmax_retriesoss2ResumableDownloadStoreresumable_store_downloadResumableStoreresumable_store_uploadr   api)selfr   r   r   r   r'   s         e/var/www/html/land-doc-ocr/venv/lib/python3.13/site-packages/modelscope/msdatasets/utils/oss_utils.py__init__OssUtilities.__init__   s    ,(" )+m6K*M&*446(#3 (,(C(C// )D )1%&*&9&9// ': '1#8    c                    SS K nU[           U l        U[           U l        U[
           U l        [        R                  " S5      S:X  a  [        U l
        OSUS    S3U l
        U[           U l        UR                  U R                  U R                  U R                  5      nUR                  X0R                  U R                  SS9U l        U[            U l        U[$           U l        g )	Nr   ENABLE_DATASET_ACCELERATIONTruezhttps://Regionz.aliyuncs.comx   )connect_timeout)r'   	ACCESS_IDkeyACCESS_SECRETsecretSECURITY_TOKENtokenr   getenvr   endpointBUCKETbucket_nameStsAuthr   bucketDIRoss_dirBACK_DIRoss_backup_dir)r-   r   r'   auths       r.   r   OssUtilities._do_init1   s    i( //
9923v=>DM&z(';&<MJDM%f-||DHHdkk4::>kk--!1!13 " H!#(2r1   c                     [         R                  S5        U R                  R                  U R                  U R
                  SU R                  S9nU R                  U5        g )Nz"Reloading sts token automatically.T)r   r   check_cookier   )loggerinfor,   !get_dataset_access_config_sessionr   r   r   r   )r-   oss_config_refreshs     r.   _reload_stsOssUtilities._reload_stsB   sS    89!XXGG**nn]]	 H $
 	()r1   c                     U(       a>  [        S[        U 5      [        U5      -  -  5      n[        SR                  U5      SSS9  g g )Nd   z{0}%  T)endflush)intfloatprintformat)consumed_bytestotal_bytesrates      r.   _percentageOssUtilities._percentageK   s?    seN3eK6HHIJD)""4(b= r1   oss_file_namedownload_configc                 .   SS K nUR                  n[        R                  R	                  U R
                  U5      n[        R                  R	                  U R                  U5      nUR                  nSnU(       aA  UR                  R                  U5      n	U	(       a  U	R                  [        R                  5      nSn
  U
S-  n
U(       a  UnO$U R                  R                  U5      (       a  UOUn[        US S9n[        R                  R	                  XL5      nUR                  (       d$  [        R                  R!                  U5      (       dR  UR#                  U R                  UUU R$                  U R&                  U R(                  U R*                  U R,                  S9   U$ ! [.         ab  nUR0                  R                  S5      S:X  a  U R3                  5         XR4                  :  a  [6        R9                  SU 35        Ue S nAOS nAff = fGMY  )	Nr   F   )etag)storemultiget_thresholdr$   progress_callbackr#   status  zFailed to download )r'   	cache_dirr   r   r   rE   rG   splitmeta_args_mapgetr	   ARGS_BIG_DATArC   object_existsr   force_downloadexistsresumable_downloadr)   r%   r$   r^   r#   	Exception__dict__rP   r&   rL   warning)r-   r`   ra   r'   rj   candidate_keycandidate_key_backuprk   big_data	args_dictretry_countfile_oss_keyfilename
local_pathes                  r.   downloadOssUtilities.downloadQ   s   #--	T\\=A!ww||D,?,?O%%'5599%@I$==)E)EFq #0L48KK4M4M%5' 5'=,@ !/4HWW\\)>
"11":$ :$++$"";;+/+C+C"&..*.*:*:$($4$4 , 6    ::>>(+s2$$&"2"22NN%8#HIG 31 s   ?C#F% %
H/AHHoss_object_namelocal_file_pathindicate_individual_progressupload_modereturnc                    SS K nSn[        R                  R                  U R                  U5      nU(       a  U R
                  nOS n  US-  nU R                  R                  U5      n	U[        R                  :X  a0  U	(       a)  [        R                  SU SUR                   S35         U$ UR                  U R                  UUU R                  U R                  U R                   UU R"                  S9   U$ ! [$         aI  n
U
R&                  R)                  S5      S:X  a  U R+                  5         X`R,                  :  a  e  S n
A
OS n
A
ff = fGM	  )	Nr   rc   zSkip z in case of z mode.)re   r%   r$   rg   r#   rh   ri   )r'   r   r   r   rE   r^   rC   ro   r
   APPENDrL   rM   valueresumable_uploadr+   r%   r$   r#   rs   rt   rm   rP   r&   )r-   r   r   r   r   r'   rz   
object_keyrg   existr~   s              r.   uploadOssUtilities.upload   sB    	WW\\$,,@
' $ 0 0 $q 11*=*"3"33KK0[=N=N<OvV $ ! %%KK#55(,(@(@"nn&7 $ 0 0 & 2    ::>>(+s2$$&"2"22 3) s    	A!C8 -AC8 8
E?EE)r,   rC   rA   r   r?   r9   r&   r%   r   r#   rG   rE   r$   r)   r    r+   r   r;   r=   N)__name__
__module____qualname____firstlineno__r/   r   rP   staticmethodr^   strr   r   boolr
   r   __static_attributes__ r1   r.   r   r      se    *3"* > >
.c ."4.`&c &C &-1&&&+.&r1   r   )
__future__r   r!   r   datasets.utils.file_utilsr   modelscope.hub.apir   .modelscope.msdatasets.download.download_configr   modelscope.utils.config_dsr   modelscope.utils.constantr   r	   r
   modelscope.utils.loggerr   rL   r8   r:   r<   r@   rF   rD   r   r   r1   r.   <module>r      s\    &  	 : % M 4C C .		 	M Mr1   