
    6pjV                        S SK rS SKrS SKrS SKJrJr  S SKrS SKr	S SK
rS SKJrJrJrJrJrJr  S SKJr  S SKJr  S SKJr  S SKJ r   S SKJr  S S	KJr  S S
KJr  S SKJ r   S SK!J"r"J#r#  S SK$J%r%  S SK&J'r'  S SK(J)r)J*r*J+r+  S SK,J-r-  \-" 5       r.Sr/Sr0 " S S\ Rb                  5      r2 " S S\25      r3 " S S\ Rb                  5      r4g)    N)DictUnion)ArrowBasedBuilderDatasetDatasetDictGeneratorBasedBuilderIterableDatasetIterableDatasetDict)is_remote_filesystem)DatasetInfo)camelcase_to_snakecase)csv)
map_nested)FileLock)HubApi)DatasetContextConfig)ExternalDatasetNativeIterableDataset)DataStreamingDownloadManager)get_subdir_hash_from_split)DEFAULT_DATASET_NAMESPACEDatasetPathNameDownloadMode)
get_logger	delimiter,c                      ^  \ rS rSrS\4U 4S jjr\4S jrSS\4S\4S jjr	S r
S	 rS
 rS rS rS\4S jrSrU =r$ )CsvDatasetBuilder(   dataset_context_configc           
      j  > UR                   U l         UR                  U l        UR                  U l        UR                  U l        UR                  U l        UR
                  U l        UR                  R                  U l        UR                  R                  U l        UR                  U l
        [        0 5      U l        [        R                  R                  U R                  U R                  U R                   U R                  [         R"                  5      U l        [&        U l        [*        U R                  ;   a  U R                  [*           U l        U R                  R-                  SS 5      U l        U R                  R-                  SS 5      U l        U R
                  =(       d-    [3        UR                  R4                  R7                  5       5      n[9        X R                  S9nSSKJnJn  U R                  RA                  5        VVs0 s H  u  pgXe" U/S S9_M     nnnURC                  U5      n[D        T	U ]  " SU R$                  U R                  UUS.U R                  D6  U R                   U RH                  l%        [M        U R                   5      U l'        [        0 5      U l(        g s  snnf )	Nengine	chunksizesplitversionr   )DataFilesDictDataFilesList)origin_metadata)	cache_dirconfig_namehash
data_files ))dataset_namecache_root_dir	namespacer&   subset_namer%   data_meta_configmeta_data_fileszip_data_filesconfig_kwargsinput_config_kwargsdictsplit_path_dictospathjoinr   	META_NAMEcache_build_dirDEFAULT_CSV_DELIMITERcsv_delimiterDELIMITER_NAMEpop
csv_enginecsv_chunksizelisttarget_dataset_structurekeysr   datasets.data_filesr'   r(   itemsfrom_local_or_remotesuper__init__infobuilder_namer   namelocal_meta_csv_paths)
selfr    r%   sub_dir_hashr'   r(   kvr-   	__class__s
            j/var/www/html/pdf-tiff/venv/lib/python3.13/site-packages/modelscope/msdatasets/download/dataset_builder.pyrL   CsvDatasetBuilder.__init__*   s:   2??4CC/99-551==+11
5FFVV4EETT#9#G#G #Bx!ww||D,?,?,0NND<M<M,0LL,;,E,E G 3T555!%!9!9.!ID 2266xF!5599+tL

 Dd#9#J#J#;#;DDFD1/ 	E ,,224
4 }aS$774 	 
 #77
C
 	(**!		(
 &&	( "&!2!2		*4+<+<=	*.r(!!
s   J/c           	      x    [         R                  R                  U R                  U R	                  SSUS95      nU$ )NFT)with_version	with_hashr1   )r:   r;   r<   _cache_dir_root_relative_data_dir)rQ   r1   builder_data_dirs      rV   _build_cache_dir"CsvDatasetBuilder._build_cache_dir[   sB    77<<  ##"di $ IJ
      Treturnc                 
   Uc  U R                   R                  OU SU R                   R                   3nU R                  nU R                  nU(       a)  [        R
                  R                  X@R                  5      nU(       a=  [        R
                  R                  U[        U R                  R                  5      5      nU(       a;  U(       a4  [        U[        5      (       a  [        R
                  R                  XF5      nU$ )zRelative path of this dataset in cache_dir:
Will be:
    self.name/self.config.version/self.hash/
or if a namespace has been specified:
    self.namespace___self.name/self.config.version/self.hash/
___)rM   rN   configr,   r:   r;   r<   	config_idstrr&   
isinstance)rQ   rY   rZ   r1   r]   builder_configr,   s          rV   r\   $CsvDatasetBuilder._relative_data_dirc   s     6?5F49911ykY\]a]f]f]s]s\tLuyy!ww||,<nnM!ww||,<,/0C0C,D F*T3"7"7!ww||,<Cr`   c           
         U R                   R                  (       d  [        S5      eUR                  U R                   R                  5      nUR                  U R                  5      n/ nUR                  5        Hb  u  pV[        U[        5      (       a  U/nUR                  [        R                  " UUR                  U5      UR                  U5      S.S95        Md     U$ )Nz7At least one data file must be specified, but got none.)filesbase_dirrO   
gen_kwargs)rd   r-   
ValueErrordownload_and_extractr5   rI   rg   rf   appenddatasetsSplitGenerator
iter_filesget)rQ   
dl_managerr-   r5   splits
split_namerk   s          rV   _split_generators#CsvDatasetBuilder._split_generatorsy   s    {{%%IK K44T[[5K5KL
#889L9LM!+!1!1!3J%%%MM''#!+!6!6u!=$2$6$6z$B  "4 r`   c           
   #     ^#    U R                   S:X  a%   [        R                  " [        R                  5        U R                  R                  b4  [        R                  " U R                  R                  R                  5      OS nU(       aC  [        UR                  UR                  5       VVs0 s H  u  pEXER                  5       _M     snnOS n[        U5       GH  u  pg[!        SUU R"                  U R$                  =(       d    SS9nU R                   b  U R                   US'   [&        R(                  " U40 UD6n	/ n
U	R*                  R                   H,  nUR-                  S5      (       d  M  U
R/                  U5        M.      [        U	5       HT  u  pU
 H%  nT(       d  M  X   R1                  U4S j5      X'   M'     [        R2                  R5                  XS	9nXl4U4v   MV     GM     g ! [
         a    [        R                  " S5         GNf = fs  snnf ! [6         a-  n[8        R;                  S
U S[        U5       SU 35        e S nAff = f7f)NpythoniTi'  )iteratordtyper   r#   r"   :FILEc                 D   > [         R                  R                  TU 5      $ Nr:   r;   r<   )xrl   s    rV   <lambda>4CsvDatasetBuilder._generate_tables.<locals>.<lambda>   s    "'',,x*Cr`   )schemazFailed to read file 'z' with error z: )rC   
csv_modulefield_size_limitsysmaxsizeOverflowErrorrd   featurespar   typezipnamestypesto_pandas_dtype	enumerater8   r@   rD   pdread_csv_engineendswithrq   applyTablefrom_pandasro   loggererror)rQ   rk   rl   r   rO   r~   file_idxfile	pd_kwargscsv_file_readertransform_fields
field_name	batch_idxdfpa_tablees     `             rV   _generate_tables"CsvDatasetBuilder._generate_tables   s    ??h&8++CKK8
 !% 4 4 @ 4;;//44 FJ 	
   #6<<>
> ''))>
  	 (.NH,,,,5	7I
 *&*oo	(# kk$<)<O!-55;;
&&w//$++J7 <%.%?MI&6
#8-/^-A-A C.EBN '7  "xx33B3FH#/99 &@ / ! 8++J78

2  +D6tAwir!MOsf   I$G1 A7I0HB"I2IH&AH+I1 HIH	I
I((IIIc                 b   UR                   R                  nUR                   R                  nU(       d  [        R                  nUR                   R
                  nU(       d  [        R                  nU R                  nU(       d  [        R                  n/ nUR                  [        R                  5        UR                  UR                   R                  5        UR                  U5        UR                  U5        UR                  U5        [        R                  R                  U5      n	[        R                  R                  UR                  [        R                  5      U	S-   5      n
[        U
5         [        R                  R!                  U5      nU(       aD  U["        R$                  R&                  :X  a&  [(        R+                  SU R,                   SU S35        [(        R/                  SU R,                   SU S35        U R1                  X!S9  S S S 5        g ! , (       d  f       g = f)N.lockReusing dataset  ()Generating dataset )rv   download_mode)download_configr*   r%   r   LOCK_FILE_NAME_ANYr&   r2   rq   DATA_FILES_NAMEr/   LOCK_FILE_NAME_DELIMITERr<   r:   r;   stripr   existsr   REUSE_DATASET_IF_EXISTSvaluer   warningrO   rM   _download_and_prepare)rQ   r   rv   download_kwargstarget_cache_dirrx   version_namer2   lock_file_nameslock_file_name	lock_pathdata_existss               rV   download_and_prepare&CsvDatasetBuilder.download_and_prepare   s    &55??//55
(;;J!1199*==L&&)<<K >>?z99FFG|,{+z*(AAFF GGLL""?#B#BCW$&	 i ''..)9:K}0T0T0Z0ZZ&tyyk4D3EQGIKK-dii[;K:LANO&&% ' D !  s   7B H  
H.c           	         SS K nUR                  R                  nU[        R                  R
                  :X  a%  UR                  USS9  [        R                  " USS9  U R                  R                  5        VVs0 s H  u  pVU[        R                  " Xd5      _M     snnU l        UR                  U R                  5      U l        g s  snnf )Nr   T)ignore_errors)exist_ok)shutilr   r*   r   FORCE_REDOWNLOADr   rmtreer:   makedirsr4   rI   r   fetch_meta_files_from_urlrP   rp   r5   r9   )rQ   rv   r   r   r   rS   rT   s          rV   r   'CsvDatasetBuilder._download_and_prepare   s    %55??L99???MM*$M?KK(48 ,,224%
4 v//DD4%
!
  *>> !%
s   ;#Cc                 \  ^ [        SU R                  S9nU R                  b  U R                  US'   [        R                  " U40 UD6n/ nUR
                  R                  5        H,  nUR                  S5      (       d  M  UR                  U5        M.     U R                  R                  US5      mU H  n[        T[        5      (       ah  [        T5      S:  aY  UR                  S   [        T5      :w  a7  [        R!                  SU SUR                  S    S	[        T5       S
35        Mz  TXF'   M  [        T["        5      (       a"  T(       a  XF   R%                  U4S j5      XF'   M  [        R'                  SU 35        M     [(        R*                  R-                  U5      n[/        US9$ )NFr}   r   r"   r    r   z,Number of lines in meta-csv file for split 'z' (z&) does not match number of data-files(z)!c                 D   > [         R                  R                  TU 5      $ r   r   )r   base_extracted_dirs    rV   r   ;CsvDatasetBuilder._convert_csv_to_dataset.<locals>.<lambda>  s    bggll+=qAr`   zNothing to do for field )arrow_table)r8   r@   rC   r   r   columnstolistr   rq   r9   ru   rg   rE   lenshaper   r   rf   r   r   r   r   r   r   )	rQ   rx   csv_file_pathr   r   r   r   pa_datar   s	           @rV   _convert_csv_to_dataset)CsvDatasetBuilder._convert_csv_to_dataset   s   %43E3EF	??&"&//Ih[[4)4**++-J""7++ ''
3 . 04/C/C/G/G0*J,   $'(:$;a$?88A;#&8"99LLFzlRUVXV^V^_`VaUb c??BCU?V>WWY[
 &8BN.449K!#!5!5A"C !9*FG +  ((&&r*7++r`   c                     [        U R                  R                  5        VVs0 s H  u  pXR                  X5      _M     snn5      $ s  snnf r   )r   rP   rI   r   )rQ   rS   rT   s      rV   
as_datasetCsvDatasetBuilder.as_dataset  sN    11779
9 ++A119
  	 
s   A
)r>   r0   rD   r@   rC   r/   r7   rP   r4   rO   r1   r%   r9   r2   r&   r5   )__name__
__module____qualname____firstlineno__r   rL   r   r^   rf   r\   ry   r   r   r   r   r   r   __static_attributes____classcell__rU   s   @rV   r   r   (   sf    /3/C /3b *C   )-%)%>  DG ,&&P%DN! ,BK  r`   r   c                   4    \ rS rSrS\4S jrS rS rS rSr	g)	TaskSpecificDatasetBuilderi  r    c                    UR                   U l        UR                  U l        UR                  U l        UR                  U l        UR
                  U l        U R                  =(       d-    [        UR                  R                  R                  5       5      n[        X R
                  S9U l        UR                  R                  U l        UR                  R                  U l        S U l        S U l        ["        R$                  " SUR                   05      U l        [(        R*                  R-                  UR.                  5      U l        U R3                  5       U l        UR                  R6                  U l        g )Nr$   rN   )r/   rO   r2   r1   r%   r&   rE   r3   rF   rG   r   r,   r4   r-   r5   r9   rd   r   	from_dictrM   r:   r;   
expanduserr0   r[   r^   
_cache_dirmeta_args_map_config_kwargs)rQ   r    r%   s      rV   rL   #TaskSpecificDatasetBuilder.__init__  s!    +77	1==/99+11
-55

 Dd#9#J#J#;#;DDFD./	0AAQQ4EETT#))3@@AC	!ww11"11 3//14EESSr`   c                 r   [         R                  R                  U R                  U R                  R                  [         R                  S5      S-   5      n[        U5         [         R                  R                  U R                  5      nU(       aN  U[        R                  :X  a:  [        R                  SU R                   SU R                   S35         S S S 5        g [        R                  SU R                   SU R                   S35        S S S 5        U R                  US9  g ! , (       d  f       N= f)N_r   r   r   r   r   )rv   )r:   r;   r<   r[   r   replacesepr   r   r   r   r   r   rO   rM   r   )rQ   r   rv   r   r   r   s         rV   r   /TaskSpecificDatasetBuilder.download_and_prepare-  s     GGLL  OO##BFFC07:<	 i ''..9K}0T0TT&tyykDOO3DAFH !  KK-dii[4??:K1MN ! 	""j"9 ! s   !A6D( 0D((
D6c                 D    UR                  U R                  5      U l        g r   )rp   r5   r9   )rQ   rv   s     rV   r   0TaskSpecificDatasetBuilder._download_and_prepare<  s    )>> !r`   c                 B    [        U R                  U R                  5      $ r   )r   r9   r   )rQ   s    rV   r   %TaskSpecificDatasetBuilder.as_dataset@  s    t33T5H5HIIr`   )r   r[   r   rd   r-   r,   rM   rO   r1   r%   r9   r2   r&   r5   N)
r   r   r   r   r   rL   r   r   r   r   r.   r`   rV   r   r     s!    T/C T.:!Jr`   r   c                      ^  \ rS rSrS\4U 4S jjr\S\S\R                  4S j5       r	S\
S\\\\4   \4   4S jrS\
4S jrS\4S	 jrS
 rS\SS4S jr\S\S\S\4S j5       rSrU =r$ )IterableDatasetBuilderiD  r    c           	      |  > UR                   U l         UR                  U l        UR                  U l        UR                  U l        UR                  U l        UR
                  U l        UR                  R                  U l        UR                  R                  U l        UR                  U l
        UR                  U l        [        R                  R                  U R                  U R                  U R                   U R                  [        R                   5      U l        [$        U l        [(        U R                  ;   a  U R                  [(           U l        U R
                  =(       d-    [+        UR                  R,                  R/                  5       5      n[1        X R                  S9n[2        TU ]h  " SU R"                  U R                   U R                  US S.U R                  D6  U R                   U R6                  l        [;        U R                   5      U l        S U l        UR                  R@                  U l         g )Nr$   )r*   r/   r+   r,   r-   r.   )!r/   r0   r1   r&   r2   r%   r3   r4   r5   r6   r7   stream_batch_sizer:   r;   r<   r   r=   r>   r?   r@   rA   rE   rF   rG   r   rK   rL   rM   rN   r   rO   meta_csv_dfmeta_cache_dir)rQ   r    r%   rR   rU   s       rV   rL   IterableDatasetBuilder.__init__F  s   2??4CC/99-551==+11
5FFVV4EETT#9#G#G !7!I!I!ww||D,?,?,0NND<M<M,0LL,;,E,E G 3T555!%!9!9.!ID

 Dd#9#J#J#;#;DDFD1/ 	 	(****	( &&	( "&!2!2		*4+<+<=	4EETTr`   ra   c                     [        U S9nU$ )N)r    )r   )r    builder_instances     rV   get_builder_instance+IterableDatasetBuilder.get_builder_instancen  s     2#9;r`   rv   c                 v   [        U [        [        45      (       d  [        SU R                   S35      e[        U R                  5      (       + nU(       d,  [        S[        U R                  5      R                   S35      eU R                  U5        U R                  U5       Vs0 s H  nUR                  U_M     nnUR                  R                  nUc  UnO$XT;   a  XE   nO[        SU S[        U5       35      e[        U R                   USS9n[        U["        5      (       a  [%        U5      nU$ s  snf )	NzBuilder z is not streamable.z(Loading a streaming dataset cached in a z is not supported yet.zBad split: z. Available splits: T)	map_tuple)rg   r   r   ro   rO   r   _fsNotImplementedErrorr   r   _check_manual_downloadry   r   r%   rE   r   _as_streaming_dataset_singler8   r
   )rQ   rv   is_localsgsplits_generatorsr%   splits_generatorstreaming_datasetss           rV   as_streaming_dataset+IterableDatasetBuilder.as_streaming_datasetu  sH    $!68I JKKx		{2EFGG+DHH55%:4>;R;R:SSij  	##J/ ,,Z8
8 GGRK8 	 
 **00=0'07eW$8>O9P8QR 
 (--

 ($//!45G!H!!1
s   &D6c           
         / nSnSnU R                   (       a,  [        [        U R                   R                  5       5      5      nU R                  (       a,  [        [        U R                  R                  5       5      5      nU(       aT  U(       dM  U R                   R                  5        H-  u  pVUR                  [        R                  " UU/ US.S95        M/     U$ U(       a  U(       a  U R                  R                  5        H`  u  pW[        U[        5      (       a  U/nU R                   R                  U5      nUR                  [        R                  " UUUUS.S95        Mb     U$ U(       dl  U(       ae  U R                  R                  5        HE  u  pW[        U[        5      (       a  U/nUR                  [        R                  " USUUS.S95        MG     U$ SU R                   S3e)Nr   )metark   rv   rm   +Neither column meta nor data file found in z#.json, specify at least one column.)r4   nextitervaluesr5   rI   rq   rr   rs   rg   rf   ru   r/   )rQ   rv   rw   meta_data_filezip_data_filerx   meta_file_urlrk   s           rV   ry   (IterableDatasetBuilder._split_generators  s   !$t';';'B'B'D"EFN d&9&9&@&@&B!CDM--1-A-A-G-G-I)
++'$1%'*4$ .JP = %)%8%8%>%>%@!
eS))"GE $ 4 4 8 8 D++'$1%**4$	 &A: !  M%)%8%8%>%>%@!
eS))"GE++'$&%**4$ &A  @@Q@Q?RRuvvr`   c                 x    U R                  U5      n[        UU R                  UR                  U R                  S9$ )N)rM   r%   r   ) _get_examples_iterable_for_splitr   rM   rO   r   )rQ   r  ex_iterables      rV   r  3IterableDatasetBuilder._as_streaming_dataset_single  s>    
 ;;<LM$"''"44	6 	6r`   c              +     #    UR                  S5      nUR                  S5      nUR                  S5      n[        5       nSnSnU(       aW  [        [        [	        U5      5      5      nUR                  S5      (       a$  Sn[        R                  R                  U5      S   nU(       aH  U(       dA  U R                  U5        [        R                  R                  U R                  5      n	SU	4v   g U(       a  U(       a  U R                  U5        U(       aB  UR                  U R                  U R                   U R"                  U5      n
XR$                  l        [        R                  R                  U R                  5      n	SU	4v   g U(       d/  U(       a(  [        R                  R)                  S	U05      n	SU	4v   g S
U R                   S3e7f)Nr  rk   rv   Fr   z.zipTr   z
Input:FILEr  z.json .)ru   r   rf   r  r  r   r:   r;   splitext_get_meta_csv_dfr   r   r   r   &get_dataset_access_config_for_unzippedr/   r1   r&   r   
oss_configfrom_pydict)rQ   rn   r  rk   rv   hub_apiis_zipzip_file_namezip_filer   oss_config_for_unzippeds              rV   r   'IterableDatasetBuilder._generate_tables  sp    "v.w'^^L1
(4U,-H  (( " 0 0 :1 =!!-0xx++D,<,<=HX+u!!-0*1*X*X%%t~~t||!+#' 9P**5xx++D,<,<=HX+5xx++\5,ABHX+ @@Q@Q?RRYZZs   GGr  Nc                     U R                   b  U R                   R                  (       aF  [        R                  " XR                  5      n[
        R                  " USU R                  S9U l         g g )NFr   )r   emptyr   r   r   r   r   r@   )rQ   r  meta_csv_file_paths      rV   r  'IterableDatasetBuilder._get_meta_csv_df  s[    #t'7'7'='=!'!A!A22"4!{{",, .D (>r`   headerstextsr   c                     0 nU R                  U5      n [        S[        U 5      5       H7  n/ nU H&  nUR                  UR                  U5      U   5        M(     XSX   '   M9     U$ )Nr   )r%   ranger   rq   )r-  r.  r   residxcol_listlines          rV   trans_data_to_mapping,IterableDatasetBuilder.trans_data_to_mapping  sd    --	*CL)CH

9 5c :;  (	 *
 
r`   )r>   r0   r@   r/   r7   r   r   r4   rO   r1   r%   r   r2   r&   r5   )r   r   r   r   r   rL   staticmethodr   Csvr   r   r   r   rf   r	   r  ry   r   r  r   r  rE   r5  r   r   r   s   @rV   r   r   D  s    &U/C &UP  $8 =@WW   &"6&"	tC()?:	;&"P1,H 1f
6 

6'[R.c .d . s 4 C  r`   r   )5r   r   r:   r   typingr   r   rr   pandasr   pyarrowr   r   r   r   r   r	   r
   datasets.filesystemsr   datasets.infor   datasets.namingr   datasets.packaged_modulesdatasets.utils.py_utilsr   filelockr   modelscope.hub.apir   4modelscope.msdatasets.context.dataset_context_configr   !modelscope.msdatasets.dataset_clsr   r   /modelscope.msdatasets.download.download_managerr   )modelscope.msdatasets.utils.dataset_utilsr   modelscope.utils.constantr   r   r   modelscope.utils.loggerr   r   rA   r?   r8  r   r   r   r.   r`   rV   <module>rI     s     	 
    + + 6 % 2 ) .  %F!F F .	 i iX-J!2 -J`SSWW Sr`   