
    9i	                     R    S SK r S SKJr  S SKJr  S SKJr  S SKJ	r	   " S S\
5      rg)    N)Pool)tqdm)OssUtilities)
UploadModec                   j    \ rS rSrS\S\S\4S jrS\S\S\S	\4S
 jrS\S\S\S\S\	S\S	\4S jr
Srg)DatasetUploadManager   dataset_name	namespaceversionc                 d    SSK Jn  U" 5       nUR                  UUSUS9n[        UUUUS9U l        g )Nr   )HubApiF)r
   r   check_cookierevision)
oss_configr
   r   r   )modelscope.hub.apir   !get_dataset_access_config_sessionr   oss_utilities)selfr
   r   r   r   _hub_api_oss_configs          h/var/www/html/land-doc-ocr/venv/lib/python3.13/site-packages/modelscope/msdatasets/utils/upload_utils.py__init__DatasetUploadManager.__init__   sI    -8@@%	 A  *"%	    object_namelocal_file_pathupload_modereturnc                 >    U R                   R                  UUSUS9nU$ )NToss_object_namer   indicate_individual_progressr   r   upload)r   r   r   r   
object_keys        r   r%   DatasetUploadManager.upload   s1    ''..'+)-#	 / %

 r   object_dir_namelocal_dir_pathnum_processes	chunksizefilter_hidden_filesc                 <  ^ ^ U U4S jn/ n[         R                  " U5       H  u  pnU H  nU(       a  UR                  S5      (       a  M"  [         R                  R	                  UU	R                  USS5      R                  S5      U5      n[         R                  R	                  X5      nUR                  X45        M     M     [        US9 n[        [        UR                  XxUS9[        U5      S95      nS S S 5        [        W5      $ ! , (       d  f       N= f)	Nc                 J   > TR                   R                  U S   U S   STS9  g )Nr      Fr!   r$   )argsr   r   s    r   
run_upload3DatasetUploadManager.upload_dir.<locals>.run_upload*   s0    %% $Q $Q-2'	 & )r   . r/   /)	processes)r+   )total)oswalk
startswithpathjoinreplacestripappend
ThreadPoollistr   imaplen)r   r(   r)   r*   r+   r,   r   r1   
files_listrootdirsfiles	file_namer   r   poolresults   `     `          r   
upload_dirDatasetUploadManager.upload_dir&   s    	) 
!#!8D"	&9+?+?+D+D ggll#LLQ7==cBIO #%'',,t"?!!;"@A # "9 -0DIIj	IJj/+,F 1 6{ 10s   +D
D)r   N)__name__
__module____qualname____firstlineno__strr   r   r%   intboolrK   __static_attributes__ r   r   r   r      sz    S S 3 #  &+.# s "%25(,;EJMr   r   )r8   multiprocessing.dummyr   r@   	tqdm.autor   %modelscope.msdatasets.utils.oss_utilsr   modelscope.utils.constantr   objectr   rU   r   r   <module>r[      s"    
 4  > 086 8r   