
    6pj                        S SK r S SKJrJr  S SKJrJr  S SKJrJ	r	J
r
JrJr  S SKJr  S SKJrJr  S SKJr  S SKJr  S S	KJr  S S
KJr  S SKJr  S SKJr  S SKJr  \" 5       r  " S S\5      r! " S S\!5      r" " S S\!5      r#g)    N)ABCabstractmethod)OptionalUnion)DatasetDatasetBuilderDatasetDictIterableDatasetIterableDatasetDict)load_dataset)HubApiModelScopeConfig)OssAuthConfig)DatasetContextConfig)DataFilesManager)ExternalDataset)DataMetaManager)DatasetFormations)
get_loggerc                   v    \ rS rSrSrS\4S jr\S 5       r\S 5       r	\S 5       r
\S 5       r\S	 5       rS
rg)BaseDownloader   z%Base dataset downloader to load data.dataset_context_configc                     Xl         g Nr   )selfr   s     i/var/www/html/pdf-tiff/venv/lib/python3.13/site-packages/modelscope/msdatasets/data_loader/data_loader.py__init__BaseDownloader.__init__   s    &<#    c                 <    [        S[        R                   S35      e)z6The entity processing pipeline for fetching the data. 'No default implementation provided for z	.process.NotImplementedErrorr   __name__r   s    r   processBaseDownloader.process   s$     "5n6M6M5NiX
 	
r!   c                 <    [        S[        R                   S35      e)Nr#   z._authorize.r$   r'   s    r   
_authorizeBaseDownloader._authorize&   s"    !5n6M6M5Nl[
 	
r!   c                 <    [        S[        R                   S35      e)Nr#   z._build.r$   r'   s    r   _buildBaseDownloader._build,   s"    !5n6M6M5NhW
 	
r!   c                 <    [        S[        R                   S35      e)Nr#   z._prepare_and_download.r$   r'   s    r   _prepare_and_download$BaseDownloader._prepare_and_download2   s#    !5n6M6M5NNef
 	
r!   c                 <    [        S[        R                   S35      e)Nr#   z._post_process.r$   r'   s    r   _post_processBaseDownloader._post_process8   s"    !5n6M6M5No^
 	
r!   r   N)r&   
__module____qualname____firstlineno____doc__r   r   r   r(   r+   r.   r1   r4   __static_attributes__ r!   r   r   r      sv    /=/C = 
 
 
 

 
 

 
 

 
 
r!   r   c                   b   ^  \ rS rSrS\4U 4S jjrS
S jrS
S jrS
S jrS
S jr	S
S jr
S	rU =r$ )OssDownloader?   r   c                 \   > [         TU ]  U5        S U l        S U l        S U l        S U l        g r   )superr   data_files_builderdatasetbuilderdata_files_managerr   r   	__class__s     r   r   OssDownloader.__init__A   s6    /0>B :> 	 26>Br!   c                     U R                  5         U R                  5         U R                  5         U R                  5         g)zSequential data fetching process: authorize -> build -> prepare_and_download -> post_process,
to keep dataset_context_config updated. N)r+   r.   r1   r4   r'   s    r   r(   OssDownloader.processK   s/     	""$r!   c                 j   [        5       R                  U R                  R                  S9n[        R
                  " 5       n[        R                  " 5       nU R                  R                  (       d  [        XUS9nO(U R                  R                  nXl	        X$l
        X4l        X@R                  l        g)zjAuthorization of target dataset.
Get credentials from cache and send to the modelscope-hub in the future. )access_token)cookies	git_token	user_infoN)r   get_cookiesr   tokenr   get_git_tokenget_user_infoauth_configr   rL   rM   rN   )r   rL   rM   rN   rS   s        r   r+   OssDownloader._authorizeT   s     (&&44:: ' <$224	$224	**66'	KK 55AAK")$-!$-!2=##/r!   c                     [        U R                  5      nUR                  5         UR                  5         UR                  U l        [	        U R                  S9U l        U R
                  R                  5       U l        g)ziSequential data files building process: build_meta -> build_data_files , to keep context_config updated. r   N)r   r   fetch_meta_filesparse_dataset_structurer   rD   get_data_files_builderrC   )r   meta_managers     r   r.   OssDownloader._buildg   si     't'B'BC%%',,.&2&I&I# #3#'#>#>#@..EEGr!   c                    U R                   R                  R                  nU R                   R                  R                  nU R                   R                  nU R                   R
                  nU R                   R                  nU R                   R                  nU R                   R                  nU R                   R                  nU R                   R                  n	U R                   R                  n
U R                   R                  nU R                   R                  nU R                  c  U(       d  SU S3eU(       aX  U[        R                   :X  aD  U(       a  ["        R%                  SU S35        ['        U4UUUUUU	U
R(                  S.UD6U l        gU R,                  R/                  U R                  5      U l        g)z.Fetch data-files from modelscope dataset-hub. Nzmeta-file: z$.py not found on the modelscope hub.z3Use trust_remote_code=True. Will invoke codes from z9. Please make sure that you can trust the external codes.)namerevisionsplitdata_dir
data_files	cache_dirdownload_mode)r   data_meta_configdataset_py_scriptdataset_formationdataset_namesubset_nameversionr^   r_   r`   cache_root_dirrb   config_kwargstrust_remote_coderC   r   hf_compatibleloggerwarninghf_load_datasetvaluerB   rD   fetch_data_files)r   rd   re   rf   rg   rh   r^   r_   r`   ra   rb   input_kwargsrk   s                r   r1   #OssDownloader._prepare_and_downloadt   s    77HHZZ 77HHZZ22??11==--55++11..7700;;
//>>	33AA22@@ 77II<<(9~-QRR!26G6U6U!U I, XB BC +!	   !%#+11	  	 DL  22CCDLr!   c                     [        U R                  [        5      (       a0  U R                  R                  R
                  U R                  l        g g r   )
isinstancerB   r   r   rc   meta_type_map
custom_mapr'   s    r   r4   OssDownloader._post_process   s6    dllO44&*&A&A&R&R&`&`DLL# 5r!   )rC   rA   rD   rB   r   )returnN)r&   r6   r7   r8   r   r   r(   r+   r.   r1   r4   r:   __classcell__rF   s   @r   r=   r=   ?   s4    C/C C>&H$La ar!   r=   c                   R   ^  \ rS rSrSrS\4U 4S jjrS rS rS r	S r
S	 rS
rU =r$ )MaxComputeDownloader   z+Data downloader for MaxCompute data source.r   c                 2   > [         TU ]  U5        S U l        g r   )r@   r   rB   rE   s     r   r   MaxComputeDownloader.__init__   s    /0r!   c                     g r   r;   r'   s    r   r(   MaxComputeDownloader.process       r!   c                     g r   r;   r'   s    r   r+   MaxComputeDownloader._authorize   r   r!   c                     g r   r;   r'   s    r   r.   MaxComputeDownloader._build   r   r!   c                     g r   r;   r'   s    r   r1   *MaxComputeDownloader._prepare_and_download   r   r!   c                     g r   r;   r'   s    r   r4   "MaxComputeDownloader._post_process   r   r!   )rB   )r&   r6   r7   r8   r9   r   r   r(   r+   r.   r1   r4   r:   rz   r{   s   @r   r}   r}      s1    5/C  r!   r}   )$osabcr   r   typingr   r   datasetsr   r   r	   r
   r   r   ro   modelscope.hub.apir   r   &modelscope.msdatasets.auth.auth_configr   4modelscope.msdatasets.context.dataset_context_configr   3modelscope.msdatasets.data_files.data_files_managerr   !modelscope.msdatasets.dataset_clsr   ,modelscope.msdatasets.meta.data_meta_managerr   modelscope.utils.constantr   modelscope.utils.loggerr   rm   r   r=   r}   r;   r!   r   <module>r      sj    
 # "+ + 4 7 @ = H 7 .	#
S #
L]aN ]a@> r!   