
    6pj#                         S SK Jr  S SKrS SKrS SKrS SKJr  S SKJr  S SK	J
r
  S SKJr  S SKJrJrJr  S SKJr  \" 5       rS	rS
rSrSrSrSrS r " S S5      rg)    )print_functionN)hash_url_to_filename)HubApi)DataDownloadConfig)MS_CACHE_HOME)"DEFAULT_DATA_ACCELERATION_ENDPOINTMetaDataFields
UploadMode)
get_loggerAccessIdAccessSecretSecurityTokenBucket	BackupDirDirc                 V   ^ ^^^^ SSK Jn  SSKJm   " UU UUU4S jSU5      nU" 5       $ )zCreate a credentials provider for oss2 with lazy import.

Returns an instance that subclasses oss2.CredentialsProvider so that the
oss2 SDK can call ``get_credentials()`` automatically when the token
expires or authentication is needed.
r   )CredentialsProvider)Credentialsc                   4   > \ rS rSrUUUU4S jrU 4S jrSrg)?_create_credential_provider.<locals>._CredentialProviderWrapper%   c                 r   > TU l         TU l        TU l        TU l        [        R
                  " 5       U l        g )N)apidataset_name	namespacerevision	threadingLock_lock)selfr   r   r   r   s    a/var/www/html/pdf-tiff/venv/lib/python3.13/site-packages/modelscope/msdatasets/utils/oss_utils.py__init__H_create_credential_provider.<locals>._CredentialProviderWrapper.__init__'   s.    DH ,D&DN$DM")DJ    c                    > U R                      U R                  R                  U R                  U R                  SU R
                  S9nT" U[           U[           U[           S9sSSS5        $ ! , (       d  f       g= f)zhoss2 SDK will call this method automatically when it finds
the token is expired or needs authentication.Fr   r   check_cookier   )access_key_idaccess_key_secretsecurity_tokenN)	r   r   !get_dataset_access_config_sessionr   r   r   	ACCESS_IDACCESS_SECRETSECURITY_TOKEN)r    
oss_configr   s     r!   get_credentialsO_create_credential_provider.<locals>._CredentialProviderWrapper.get_credentials.   sl     !XXGG!%!2!2"nn!&!]]	 H ,
 #",Y"7&0&?#-n#= s   AA//
A=)r   r   r   r   r   N)__name__
__module____qualname____firstlineno__r"   r0   __static_attributes__)r   r   r   r   r   s   r!   _CredentialProviderWrapperr   %   s    	* 	*	 	r$   r7   )oss2r   oss2.credentialsr   )r   r   r   r   r   r7   r   s   ````  @r!   _create_credential_providerr:      s(     ), %8 2 &''r$   c            
       b    \ rS rSrSrS r\S 5       rS\S\	S\4S jr
S	\S
\S\S\S\4
S jrSrg)OssUtilitiesA   zX
A utility class for handling Alibaba Cloud OSS operations such as upload and download.
c                    Xl         X l        X0l        [        5       U l        U R                  R                  U R                   U R                  SU R                  S9n[        R                  " S5      S:X  a  [        U l	        OSUS    S3U l	        [        R                  R                  [        S5      U l        [        R                  " 5       U l        S	U l        S
U l        SU l        SSKnUR+                  U R                  S9U l        UR/                  U R                  S9U l        [3        U R                  U R                   U R                  U R                  S9nUR5                  U5      nU[6           U l        UR;                  UU R                  U R8                  US   R=                  S5      S9U l        U[@           U l!        U[D           U l#        g)zX
Initializes the OssUtilities with the given OSS configuration and dataset information.
Fr&   ENABLE_DATASET_ACCELERATIONTruezhttps://Regionz.aliyuncs.comztmp/resumable_storei   i      r   N)root)r   r   r   r   zoss-)authendpointbucket_nameregion)$r   r   r   r   r   r+   osgetenvr   rE   pathjoinr   resumable_store_root_pathmultiprocessing	cpu_countnum_threads	part_sizemultipart_thresholdmax_retriesr8   ResumableDownloadStoreresumable_store_downloadResumableStoreresumable_store_uploadr:   ProviderAuthV4BUCKETrF   r   lstripbucketDIRoss_dirBACK_DIRoss_backup_dir)r    r   r   r   r/   r8   credential_providerrD   s           r!   r"   OssUtilities.__init__F   s    )" 8XX??**nn]]	 @ $
 9923v=>DM&z(';&<MJDM)+m6K*M&*446(#3 (,(C(C// )D )1%&*&9&9// ': '1# :**nn]]	$
 ""#67%f-kk]]((h'..v6	 " 
 "#(2r$   c                     U(       a>  [        S[        U 5      [        U5      -  -  5      n[        SR                  U5      SSS9  g g )Nd   z{0}%  T)endflush)intfloatprintformat)consumed_bytestotal_bytesrates      r!   _percentageOssUtilities._percentagey   s?    seN3eK6HHIJD)""4(b= r$   oss_file_namedownload_configreturnc                 
   UR                   n[        R                  R                  U R                  U5      n[        R                  R                  U R
                  U5      nUR                  nSnU(       aA  UR                  R                  U5      nU(       a  UR                  [        R                  5      nSn	SSKn
  U(       a  UnO$U R                  R                  U5      (       a  UOUn[        USS9n[        R                  R                  X<5      nUR                  (       d$  [        R                  R!                  U5      (       dR  U
R#                  U R                  UUU R$                  U R&                  U R(                  U R*                  U R,                  S9   U$ ! [.         aU  n[0        R3                  SU SU S35        U	S	-  n	XR4                  :  a  [0        R7                  S
U S35        Ue SnAOSnAff = fGMG  )a  
Downloads a file from OSS to the local cache.

Args:
    oss_file_name (str): The name of the file in OSS to download.
    download_config (DataDownloadConfig): Configuration for the download process.

Returns:
    str: The local path to the downloaded file.
Fr   N)etag)rZ   keyfilenamestoremultiget_thresholdrP   progress_callbackrO   zError downloading : , trying again...   zFailed to download  due to exceeded retries.)	cache_dirrH   rJ   rK   r\   r^   splitmeta_args_mapgetr	   ARGS_BIG_DATAr8   rZ   object_existsr   force_downloadexistsresumable_downloadrT   rQ   rP   rm   rO   	ExceptionloggerwarningrR   error)r    ro   rp   r}   candidate_keycandidate_key_backupr~   big_data	args_dictretry_countr8   file_oss_keyru   
local_pathes                  r!   downloadOssUtilities.download   s    $--	T\\=A!ww||D,?,?O%%'5599%@I$==)E)EF#0L48KK4M4M%5' 5'=,@ !/4HWW\\)>
"11":$ :$++#{{(!+";;+/+C+C"&..*.*:*:$($4$4 , 6    (r!<MNPq "2"22LL-m_<UV G	 3	/ s   ?CF   
G?*AG::G?oss_object_namelocal_file_pathindicate_individual_progressupload_modec                    Sn[         R                  R                  U R                  U5      nU(       a  U R                  nOSnSSKn  U R                  R                  U5      n	U[        R                  :X  a0  U	(       a)  [        R                  SU SUR                   S35         U$ UR                  U R                  UUU R                  U R                  U R                   UU R"                  S9   U$ ! [$         aU  n
[        R'                  SU SU
 S	35        US
-  nXPR(                  :  a  [        R+                  SU S35        U
e Sn
A
OSn
A
ff = fGM  )ar  
Uploads a local file to OSS.

Args:
    oss_object_name (str): The name of the object in OSS.
    local_file_path (str): The local file path to upload.
    indicate_individual_progress (bool): Whether to show individual progress.
    upload_mode (UploadMode): The upload mode (e.g., OVERWRITE, APPEND).

Returns:
    str: The OSS object key where the file is uploaded.
r   NzSkip z in case of z mode.)rZ   rt   ru   rv   rQ   rP   rx   rO   zError uploading ry   rz   r{   zFailed to upload r|   )rH   rJ   rK   r\   rm   r8   rZ   r   r
   APPENDr   infovalueresumable_uploadrV   rQ   rP   rO   r   r   rR   r   )r    r   r   r   r   r   
object_keyrx   r8   existr   s              r!   uploadOssUtilities.upload   s`    WW\\$,,@
' $ 0 0 $11*=*"3"33KK0[=N=N<OvV , ) %%;;",55(,(@(@"nn&7 $ 0 0 & 2    &&7r!<MNPq "2"22LL+O+<<UV G	 3	' s!   	AC3 (AC3 3
E=AEE)r   rZ   rF   r   rE   rR   rQ   r   rO   r^   r\   rP   rT   rL   rV   r   N)r2   r3   r4   r5   __doc__r"   staticmethodrm   strr   r   boolr
   r   r6    r$   r!   r<   r<   A   si    13f > >
;c ;"4;9<;z6c 6C 6-16&6+.6r$   r<   )
__future__r   rM   rH   r   datasets.utils.file_utilsr   modelscope.hub.apir   .modelscope.msdatasets.download.download_configr   modelscope.utils.config_dsr   modelscope.utils.constantr   r	   r
   modelscope.utils.loggerr   r   r,   r-   r.   rX   r]   r[   r:   r<   r   r$   r!   <module>r      se    &  	  : % M 4C C .		 	#(Lq qr$   