
    <>i$                        S SK r S SKrS SKrS SKrS SKr S SKrS SKrS SK	J
r
  S SKrS SKJr  S SKJrJrJrJrJrJr  S SKJr  S SKJr   S SKJr  S SKJrJ r J!r!J"r"   S SK#r$S SK%J&r'  \RP                  R8                  \RP                  RR                  /r*S r+\RP                  RF                  S 5       r,\RP                  RF                  S 5       r-\RP                  RF                  S	 5       r.\RP                  RF                  S
 5       r/\RP                  Ra                  \1\24SS9\RP                  RF                  S 5       5       r3\RP                  RF                  S 5       r4\RP                  RF                  S 5       r5\RP                  RF                  S 5       r6\RP                  RF                  S 5       r7\RP                  RF                  S 5       r8\RP                  RF                  \RP                  Rs                  SS/S//\Rt                  " S5      S:  \Rt                  " SS5      S:  \Rt                  " SS5      Rw                  \Rx                  " 5       5      S:  45      \RP                  Rs                  SS5      S 5       5       5       r=\RP                  RF                  S 5       r>\RP                  R~                  S 5       r@\RP                  R~                  S 5       rA\RP                  RF                  S  5       rB\RP                  RF                  \RP                  R~                  S! 5       5       rCS" rDS# rES$ rF\RP                  RF                  S% 5       rG\RP                  RF                  S& 5       rH\RP                  RF                  S' 5       rI\RP                  R
                  S( 5       rJ\RP                  R
                  S) 5       rK\RP                  R
                  S* 5       rLSRS+ jrMS, rN\RP                  R
                  \RP                  Rs                  S-S.S//5      S0 5       5       rO\RP                  R
                  S1 5       rP\RP                  R
                  S2 5       rQ\RP                  R
                  \RP                  Rs                  S-S.S//5      S3 5       5       rRS4 rSS5 rT   SSS6 jrU STS7 jrV\RP                  RF                  S8 5       rW\RP                  RF                  S9 5       rX\RP                  RF                  S: 5       rY\RP                  RF                  S; 5       rZ\RP                  RF                  S< 5       r[\RP                  RF                  \RP                  R~                  S= 5       5       r\\RP                  RF                  \RP                  R~                  \RP                  R                  \R                  S>:H  S?S@9SA 5       5       5       r_\RP                  RF                  \RP                  R~                  SB 5       5       r`\RP                  RF                  SC 5       raSUSD jrb\RP                  RF                  SE 5       rc\RP                  RF                  SF 5       rdSG reSH rfSI rgSJ rhSK riSL rjSM rk\RP                  Rs                  SNSO5      SP 5       rl\RP                  RF                  SQ 5       rmg! \ a    Sr GNf = f! \ a    Sr GNlf = f! \ a    S=r$r' GNrf = f)V    N)FileSelector
FileSystemLocalFileSystemPyFileSystemSubTreeFileSystemFSSpecHandler)util)guid)_read_table_test_dataframe_test_table_write_tablec                    [         R                  " S/ SQ05      nU S-  nUR                  5         US-  n[        R                  " U[        U5      5        [        R                  " U[        5       S9nUR                  U5      (       d   e[        R                  " S[        R                  " U 5      S9nUR                  U5      (       d   eg )Na         data_dirdata.parquet
filesystemzdata_dir/data.parquet)patablemkdirpqwrite_tablestr
read_tabler   equalsr	   _filesystem_uri)tempdirr   	directorypathresults        a/var/www/html/land-tabula/venv/lib/python3.13/site-packages/pyarrow/tests/parquet/test_dataset.pytest_filesystem_urir'   ;   s    HHc9%&E*$IOO~%DNN5#d)$ ]]*,F== ]]D,@,@,IKF==    c                 .    [        5       n[        X5        g N)r   _partition_test_for_filesystem)r"   locals     r&   test_read_partitioned_directoryr-   N   s    E"52r(   c                     [        5       nU n[        X5        [        R                  " U5      nUR	                  S/S9nUR
                  S/:X  d   eg )Nvaluescolumns)r   r+   r   ParquetDatasetreadcolumn_names)r"   r,   	base_pathdatasetr%   s        r&   'test_read_partitioned_columns_selectionr7   T   sQ     EI"54	*G\\8*\-F8*,,,r(   c                 `   [        5       nU nSS/n/ SQnSS/nSU/SU/SU//n[        R                  " [        R                  " US	S
9R                  S5      [        R                  " [        R                  " [        R                  " U[        S
9S5      S5      [        R                  " [        R                  " [        R                  " USS
9S5      S5      [        R                  " S5      S.5      n[        XXg5        [        R                  " X!/ SQS9nUR                  5       n	U	R                  5       R                  SS9n
SU
S   R                  ;  d   eSU
S   R                  ;  d   eSU
S   R                  ;  d   e/ SQSS//n[        R                  " X!US9nUR                  5       n	U	R                  5       R                  SS9n
U
S   S:H  U
S   S:g  -  U
S   S:H  -  n[        R                  " U
S   5      S:H  U
S   S:H  -  nUR!                  5       S:  d   eUR!                  5       S:  d   eU
R"                  S   UR!                  5       UR!                  5       -   :X  d   eS//S//4 H8  n[        R                  " X!US9nUR                  5       R$                  S:X  a  M8   e   g )Nr   r   r   bcTFintegerstringbooleani4dtype      r   boolr      r<   r=   r>   r/   ))r<   =r   )r=   !=r:   )r>   ==Truer   filtersdropr:   )r<   rG   r   )r>   rI   FalserJ   rO   )r=   rI   s   1 a)r=   rI   z1 a)r   pd	DataFramenparrayrepeattileobjectarange_generate_partition_directoriesr   r2   r3   	to_pandasreset_indexr/   sumshapenum_rows)r"   r,   r5   integer_keysstring_keysboolean_keyspartition_specdfr6   r   	result_dfrL   df_filter_1df_filter_2s                 r&   test_filters_equivalencyrf   a   s   EIq6L!K%=L	L!	;	L!N 
88L5<<R@''"''"((;f"EqI1M77277288L#GKQO))B-	 
B $EnI ,G
 LLNE"..D.9IIi(/////i)00000	),33333	

 
89G W6GLLNE!--4-8I Y'1,X#%'Y6)+K 88Ii01Q6Y7*,K??q   ??q   ??1+//"3koo6G"GHHH/01./02##:||~&&!+++	2r(   c                    [        5       nU n/ SQnSU//nSn[        R                  " [        R                  " U5      [        R
                  " USS9S.SS/S9n[        XXF5        [        R                  " X!S	S
/S9nUR                  5       nUR                  5       R                  SS9R                  SS9n	[        [        U	S   R                  5       V
s/ s H  oPM     nn
USS/:X  d   eg s  sn
f )Nr   r   r   r      integersrC   r?   r@   indexrj   rl   r0   )rj   <ri   )rj   >r   rK   byTrM   r   r   r   rP   rQ   rR   rW   rS   rX   r   r2   r3   rY   sort_valuesrZ   mapintr/   r"   r,   r5   r^   ra   Nrb   r6   r   rc   xresult_lists               r&   %test_filters_cutoff_exclusive_integerry      s    EI"L	\"N 	
A	1HH\6 $
&B
 $EnI  
G LLNE""{g{."{{-  "#y'<'C'CDED1DKE1a&    Fs   C-z5Loss of type information in creation of categoricals.)raisesreasonc           	         [        5       nU n[        R                  " SSS5      [        R                  " SSS5      [        R                  " SSS5      [        R                  " SSS5      [        R                  " SSS5      /nSU//nS	n[        R                  " [
        R                  " U5      [
        R                  " US
S9S.SS/S9n[        XXF5        [        R                  " X!SS/S9nUR                  5       nUR                  5       R                  SS9R                  SS9n	[        R                  " [
        R                  " [        R                  " SSS5      /S
S9[
        R                  " US
S9S9n
U	S   R                   U
:X  d   eg )Ni  ri   	   
            datesrC   
datetime64r@   )rl   r   rl   r0   )r   rm   z
2018-04-12)r   rn   z
2018-04-10rK   ro   TrM   
categories)r   datetimedaterP   rQ   rR   rW   rS   rX   r   r2   r3   rY   rr   rZ   Categoricalr/   )r"   r,   r5   	date_keysra   rv   rb   r6   r   rc   expecteds              r&   &test_filters_cutoff_exclusive_datetimer      sm    EI 	dAq!dAr"dAr"dAr"dAr"I 
)N 	
A	1)<8 !
#B
 $EnI((
G LLNE""{g{."{{-  ~~
(--a,-\B88I\:<H W$$000r(   c           
      B   U S-  n[         R                  " [         R                  " SSSS9[        S5      S.5      R	                  USS9  [
        R                  " US	S
[        R                  " SSS5      4/S9nUR                  S5      R                  5       / SQ:X  d   eg )Nztimestamps.parquetz
2020-01-01r~   D)periodsfreq)r   idT)use_deprecated_int96_timestampsr   <=i  r   rC   rL   r   rh   )
rP   rQ   
date_rangerange
to_parquetr   r   r   column	to_pylist)r"   r$   r   s      r&   test_filters_inclusive_datetimer      s     ))DLL|RcBBi  z$z=MM$	$))$156) E <<'')_<<<r(   c                    [        5       nU n/ SQnSU//nSn[        R                  " [        R                  " U5      [        R
                  " USS9S.SS/S9n[        XXF5        [        R                  " X!S	S
/S9nUR                  5       nUR                  5       R                  SS9R                  SS9n	[        [        U	S   R                  5       V
s/ s H  n
[        U
5      PM     nn
USS/:X  d   eg s  sn
f )Nrh   rj   rC   r?   r@   rk   rl   r0   )rj   r   r   )rj   z>=r   rK   ro   TrM   r   r   rq   ru   s               r&   test_filters_inclusive_integerr     s   EI"L	\"N 	
A	1HH\6 $
&B
 $EnI!!
G LLNE"++)+4+(  $'sIj,A,H,H#IJ#Ia3q6#IKJ1a&    Ks   C7c                    [        5       nU nSS/n/ SQnSS/nSU/SU/SU//n[        R                  " [        R                  " US	S
9R                  S5      [        R                  " [        R                  " [        R                  " U[        S
9S5      S5      [        R                  " [        R                  " [        R                  " USS
9S5      S5      [        R                  " S5      S.5      n[        XXg5        [        R                  " X!S/S9nUR                  5       n	U	R                  5       R                  SS9n
SU
S   R                  ;   d   eSU
S   R                  ;   d   eSU
S   R                  ;  d   e[        R                  " X!SSS/4SSSS14/S9nUR                  5       n	U	R                  5       R                  SS9n
SU
S   R                  ;  d   eSU
S   R                  ;  d   eSU
S   R                  ;  d   eg )Nr   r   r9   TFr<   r=   r>   r?   r@   rB   rC   r   rD   r   rE   rF   )r=   inabrK   rM   r   r:   r;   r   )r=   r   r   r:   znot inrO   )r   rP   rQ   rR   rS   rT   rU   rV   rW   rX   r   r2   r3   rY   rZ   r/   )r"   r,   r5   r^   r_   r`   ra   rb   r6   r   rc   s              r&   test_filters_inclusive_setr   .  s   EIq6L!K%=L	L!	;	L!N 
88L5<<R@''"''"((;f"EqI1M77277288L#GKQO))B-	 
B $EnI'(G LLNE"..D.9I)H%,,,,,)H%,,,,,i)00000TA3')EXy13G
 LLNE"..D.9IIi(/////i)00000	),33333r(   c                 \   [        5       nU n/ SQnSU//nSn[        R                  " [        R                  " U5      [        R
                  " USS9S.SS/S9n[        XXF5        [        R                  " [        5         [        R                  " UUS	/S
9  S S S 5        [        R                  " [        5         [        R                  " UUS/S
9  S S S 5        [        R                  " UUSS[        5       4/S
9nUR                  5       R                  S:X  d   e[        R                  " UUSSS14/S
9n[        R                  " [         5         UR                  5       R                  S:X  d   e S S S 5        g ! , (       d  f       N= f! , (       d  f       N= f! , (       d  f       g = f)Nrh   rj   rC   r?   r@   rk   rl   r0   )rj   r   r   rK   )rj   z=<r   r   r   rH   r   )r   rP   rQ   rR   rW   rS   rX   pytestrz   	TypeErrorr   r2   
ValueErrorsetr3   r]   NotImplementedError)r"   r,   r5   r^   ra   rv   rb   r6   s           r&   test_filters_invalid_pred_opr   ]  sv   EI"L	\"N 	
A	1HH\6 $
&B
 $EnI	y	!
)%*#8";	= 
"
 
z	"
)%*#8";	= 
# 	+0*4dCE)B(EGG <<>""a'''	+0*4dQC)@(ACG 
*	+||~&&!+++ 
,	+' 
"	!
 
#	" 
,	+s$   ?E;9F!F;
F	
F
F+c                 ~   [        5       nU n/ SQnSU//nSn[        R                  " [        R                  " U5      [        R
                  " USS9S.SS/S9n[        XXF5        S	n[        R                  " [        US
9   [        R                  " X!S/S9R                  5         S S S 5        g ! , (       d  f       g = f)Nrh   rj   rC   r?   r@   rk   rl   r0   z1No match for FieldRef.Name\(non_existent_column\)match)non_existent_columnrm   r   rK   )r   rP   rQ   rR   rW   rS   rX   r   rz   r   r   r2   r3   )r"   r,   r5   r^   ra   rv   rb   msgs           r&   test_filters_invalid_columnr     s     EI"L!<01N	A	1HH\6 $
&B
 $EnI
>C	z	-
)#B"E	GGKtv 
.	-	-s    %B..
B<rL   )rj   rm   r   rj   r   nestedr   r:   read_method)r   read_pandasc                    [        [        U5      n[        5       nU n/ SQnSU//n[        U5      n[        R
                  " [        R                  " U5      [        R                  " USS9[        R                  " [        U5       V	s/ s H  o[        U	5      S.PM     sn	5      S.5      n
[        XEXz5        [        XAS9nU" U40 UD6nUR                  S:X  d   eg s  sn	f )	Nrh   rj   r?   r@   r   )rl   rj   r   rK   r   )getattrr   r   lenrP   rQ   rR   rW   rS   r   r   rX   dictr]   )r"   rL   r   r3   r,   r5   r^   ra   rv   irb   kwargsr   s                r&   test_filters_read_tabler     s     2{#DEI"L	\"N 	LA	1HH\6((58D8a#a&18DE 
B $EnIU4F%f%E>>Q Es   Cc                 b   [        5       nU nSS/nSU//nSn[        R                  " [        R                  " U5      [        R
                  " USS9S.SS/S	9n[        XXF5        [        R                  " U5      nUR                  5       nUR                  S5      R                  5       U:X  d   eg )
N2019_22019_3	year_weekr   rV   r@   )rl   r   rl   r0   )r   rP   rQ   rR   rW   rS   rX   r   r2   r3   r   r   )	r"   r,   r5   r_   ra   rv   rb   r6   r%   s	            r&   $test_partition_keys_with_underscoresr     s     EIX&K	k"N 	
A	1XXk: %
'B
 $EnI	*G\\^F==%//1[@@@r(   c                     U u  pUS-   n[         R                  " S/ SQ05      n[        X2US9  [        X!S9nUR	                  U5      (       d   eg Nz/test.parquetr   r   r   r   r   r   r   r    )s3_example_s3fsfsr$   r   r%   s        r&   test_read_s3fsr     sO    HB/!DHHc9%&E,-F==r(   c                     U u  pUS-   n[         R                  " S/ SQ05      n[        XCUS9  [        X!S9nUR	                  U5      (       d   eg r   r   )r   r   r#   r$   r   r%   s         r&   test_read_directory_s3fsr     sO    #MB&DHHc9%&E,2F==r(   c                     [        U S-  5      n[        R                  " S/ SQ05      n[        X!5        [        R
                  " U/5      R                  5       nUR                  U5      (       d   eg )Nr   r   r   )r   r   r   r   r   r2   r3   r    )r"   	data_pathr   r%   s       r&   test_read_single_file_listr     s\    Gn,-IHHc9%&E"	{+002F==r(   c                 "    U u  p[        X5        g r*   )r+   r   r   r$   s      r&   $test_read_partitioned_directory_s3fsr     s     HB"2,r(   c                 v   SS/n/ SQnSU/SU//nSn[         R                  " [        R                  " U5      [        R                  " USS9R                  S	5      [        R                  " [        R                  " [        R                  " U[        S9S
5      S5      [        R                  R                  U5      S./ SQS9n[        XXF5        [        R                  " XS9nUR                  5       nUR                  5       R                  SS9R!                  SS9n	UR                  SS9R!                  SS9R#                  U	R$                  S9n
U
S   R'                  S5      U
S'   U
S   R'                  S5      U
S'   U	R$                  / SQ:H  R)                  5       (       d   e[*        R,                  " X5        g )Nr   r   r9   foobarrE   r?   r@   rB   rC   r   )rl   r   r   r/   r0   r   rl   ro   TrM   category)rl   r/   r   r   )rP   rQ   rR   rW   rS   rT   rU   rV   randomrandnrX   r   r2   r3   rY   rr   rZ   reindexr1   astypealltmassert_frame_equal)r   r5   foo_keysbar_keysra   rv   rb   r6   r   rc   expected_dfs              r&   r+   r+      s   1vHH		N 	A	1xx-44R8wwrwwrxx?CQG))//!$	
 1
2B $B>F	9GLLNE"++)+4+(  >>W>-KTK*GI$5$5G6  %U+22:>K$U+22:>K!BBGGIIII)1r(   c           	         ^ ^^^^^ [        T [        5      (       d  [        [        T 5      5      m [	        T5      m[        T S[        T SS5      5      mUUUU UU4S jmT" US/ 5        g )Npathsepsep/c                   > TU   u  p4U GHx  nX#U4/-   nTR                  [        U 5      U SU 3/5      nTR                  U5        UTS-
  :X  a  SSKJn  TR                  U[        5       /5      n	[        TU5      n
[        R                  R                  U
5      nTR                  U	5       n[        X5        S S S 5        TR                  U	5      R                  UR                  :w  d   eTR                  U	5      R                  UR                  :X  d   eTR                  US/5      nTR                  U5       n S S S 5        GM>  T" XqS-   U5        TR                  US/5      nTR                  U5       n S S S 5        GM{     g ! , (       d  f       N= f! , (       d  f       GM  = f! , (       d  f       GM  = f)NrG   r   r   )FileType_SUCCESS)joinr   
create_dir
pyarrow.fsr   r
   _filter_partitionr   Tablefrom_pandasopen_output_streamr   get_file_infotypeNotFoundFile)base_dirlevel	part_keysnamer/   valuethis_part_keys	level_dirr   	file_pathfiltered_df
part_tableffile_successDEPTH_visit_levelrb   r   ra   r   s                 r&   r   5_generate_partition_directories.<locals>._visit_level2  s   %e,E&-8NH&%!& I MM)$	!/#LL)TV)<=	/NCXX11+>
**95 / 6''	2778;L;LLLL''	2778==HHH&||Y
,CD**<8A 98 Y	>B&||Y
,CD**<8A 987   65 98
 98s$   .F	<F9F-	
F	
F*	-
F=	r   )
isinstancer   r   r   r   r   )r   r   ra   rb   r   r   r   s   ` ``@@@r&   rX   rX   '  s[     b*%%-+,Eb)WR%<=G @ 1b!r(   c                 D   [         R                  " [        U 5      [        S9n/ nU He  u  pEUR	                  U5        [        U[        R                  [        R                  45      (       a  [        R                  " U5      nX U   U:H  -  nMg     X   R                  USS9$ )Nr@   r   )axis)rR   onesr   rD   appendr   r   r   rP   	TimestamprN   )rb   r   	predicateto_dropr   r   s         r&   r   r   U  s    Bt,IG t ehmmX->->?@@LL'EX&&	 ! =gA..r(   c                 B   U S-  nUR                  5         [        R                  R                  [        R
                  " S/ SQ05      5      n[        R                  " X!S-  5        U S-  nUR                  5         [        R                  R                  [        R
                  " S/ SQ05      5      n[        R                  " XCS-  5        [        R                  " U S//S9nUR                  S5      R                  [        R                  " / SQ/5      5      (       d   eg )	NzA=0Br   r   zA=1r9   )ArI   r   r   )r   r   r   r   rP   rQ   r   r   r   r   r    chunked_array)r"   dir1table1dir2table2r   s         r&   "test_filter_before_validate_schemar
  e  s     U?DJJLXX!!",,Y/?"@AFNN6.01U?DJJLXX!!",,_/E"FGFNN6.01 MM'^,<+=>E<<##B$4$4i[$ABBBBr(   c                 t   SnSnU [        5       -  nUR                  5         / n/ n[        U5       H  n[        X&S9nUS   R	                  [
        R                  5      US'   X6 S3-  n[        R                  R                  U5      n	[        X5        UR                  U	5        UR                  U5        M     US-  R                  5         SS jn
U
" U5      n[        R                  " U5      nUR                  U5      (       d   eS	S
SUR                  S-
  /nU Vs/ s H  okR!                  U5      R"                  PM     nn[$        R&                  " X>S9n[        R                  R)                  U Vs/ s H  okR+                  U5      PM     snUUR,                  R.                  S9nUR                  U5      (       d   e[$        R&                  " USS9  [        UWS9R0                  S S 2S S24   nU [        5        S3-  n[        R                  R                  U5      n[        UU5        g s  snf s  snf )Nr~   rC   seeduint32.parquetz_SUCCESS.crcTc                 N    [         R                  " U 40 UD6nUR                  XS9$ )N)r1   use_threads)r   r2   r3   )pathsr1   r  r   r6   s        r&   read_multiple_files5test_read_multiple_files.<locals>.read_multiple_files  s(    ##E4V4||G|EEr(   r   r      r   r0   )namesmetadata)r  ri   )NT)r
   r   r   r   r   rR   int64r   r   r   r   r   touchconcat_tablesr    num_columnsfieldr   r   r   from_arraysr   schemar  iloc)r"   nfilessizedirpath	test_datar  r   rb   r$   r   r  r%   r   to_read	col_namesout	bad_applebad_apple_pathts                      r&   test_read_multiple_filesr*  {  s   FDGMMOIE6]T* (|**288483h'$$R(U!T  ~$$&F !'F	*H=="""" !Q**Q./G/67w!a%%wI7
--
3Cxx##w$Gw!]]1%5w$G*3-3]]-C-C $ EH ::h MM't,  1-221bqb59I$&22N
Y'AN# 8$Gs   $H05H5c                    SnSnU [        5       -  nUR                  5         / n/ n/ n[        U5       H  n[        X'S9n[        R
                  " Xr-  US-   U-  5      Ul        SUR                  l        X7 S3-  n	[        R                  R                  U5      n
[        X5        UR                  U
5        UR                  U5        UR                  U	5        M     [        R                  " U5      nSS/nUR                  US9R!                  5       n["        R$                  " U Vs/ s H  oU   PM	     sn5      n[&        R(                  " X5        UR                  [+        U5      S9R!                  5       nUR,                  UR,                  :X  d   e[&        R(                  " UR/                  UR0                  S9U5        g s  snf )	NrC   r  r   rl   r  uint8stringsr0   )r
   r   r   r   rR   rW   rl   r   r   r   r   r   r   r   r2   r   rY   rP   concatr   r   r   r\   r   r1   )r"   r   r!  r"  r#  framesr  r   rb   r$   r   r6   r1   r%   rw   r   s                   r&   test_dataset_read_pandasr0    s   FDGMMOIFE6]T*99QXA~63h'$$R(U!bT  (G	"G   1;;=Fyyf5fG*f56H&+   W 6@@BF<<8>>)))&..1A1A.BHM 6s   'Gc                     U [        5       -  nUR                  5         [        SSS9nUS-  n[        X#SS9  [        R
                  " USS9nUR                  5       R                  U5      (       d   eg )	Nr~   r   r  	0.parquet2.6versionT)
memory_map)r
   r   r   r   r   r2   r3   r    )r"   r"  r   r$   r6   s        r&   test_dataset_memory_mapr7    sk     GMMO#E[ De,D"G<<>  ''''r(   c                    U [        5       -  nUR                  5         [        SSS9nUS-  n[        X#SS9  [        R
                  " [        5         [        R                  " USS9  S S S 5        S	 H=  n[        R                  " XS9nUR                  5       R                  U5      (       a  M=   e   g ! , (       d  f       NR= f)
Nr~   r   r  r2  r3  r4  i)buffer_size)   i   )r
   r   r   r   r   rz   r   r   r2   r3   r    )r"   r"  r   r$   r9  r6   s         r&   #test_dataset_enable_buffered_streamr;    s    GMMO#E[ De,	z	"
	& 
# ###.||~$$U++++ #	 
#	"s   B33
Cc                 N   U [        5       -  nUR                  5         [        SSS9nUS-  n[        X#SS9  S Hi  n[        R
                  " XS9nUR                  5       R                  U5      (       d   e[        R                  " XS9nUR                  U5      (       a  Mi   e   g )	Nr~   r   r  r2  r3  r4  )TF)
pre_buffer)	r
   r   r   r   r   r2   r3   r    r   )r"   r"  r   r$   r=  r6   actuals          r&   test_dataset_enable_pre_bufferr?    s    GMMO#E[ De,#
##,||~$$U++++w>}}U#### $r(   c                     / n/ n[        U5       H>  n[        X%S9nX S3-  nUR                  [        Xg5      5        UR                  U5        M@     U$ )Nr  r  )r   r   r   r   )r5   r   
file_nrowsr#  r  r   r   r$   s           r&   _make_example_multifile_datasetrB    sX    IE6]J/S>)e23T  Lr(   c                     U Vs/ s H  n[        UR                  5       5      PM     nn[        U5      [        U R                  5      :X  d   eg s  snf r*   )r   as_posixr   files)r6   r  r$   s      r&   _assert_dataset_pathsrF  (  sA    .34edS!eE4u:W]]++++ 5s   #A
dir_prefix_.c                     U [        5       -  nUR                  5         [        USSS9nX! S3-  R                  5         [        R                  " U5      n[        XC5        g )Nr~   rC   r   rA  stagingr
   r   rB  r   r2   rF  r"   rG  r"  r  r6   s        r&   test_ignore_private_directoriesrO  -  sZ     GMMO+GB78:E W%%,,.(G')r(   c                    U [        5       -  nUR                  5         [        USSS9nUS-  R                  S5       nUR	                  S5        S S S 5        US-  R                  S5       nUR	                  S5        S S S 5        [
        R                  " U5      n[        XB5        g ! , (       d  f       N^= f! , (       d  f       NA= f)Nr~   rC   rK  z	.DS_Storewbs	   gibberishz.privater
   r   rB  openwriter   r2   rF  r"   r"  r  r   r6   s        r&   test_ignore_hidden_files_dotrV  >  s    GMMO+GB78:E K
	%	%d	+q	 
, J
	$	$T	*a	 
+ (G') 
,	+ 
+	*   B'+B8'
B58
Cc                    U [        5       -  nUR                  5         [        USSS9nUS-  R                  S5       nUR	                  S5        S S S 5        US-  R                  S5       nUR	                  S5        S S S 5        [
        R                  " U5      n[        XB5        g ! , (       d  f       N^= f! , (       d  f       NA= f)Nr~   rC   rK  _committed_123rQ  s   abcd_started_321rR  rU  s        r&   #test_ignore_hidden_files_underscorer[  Q  s    GMMO+GB78:E $
$	*	*4	0A	 
1 N
"	(	(	.!	 
/ (G') 
1	0 
/	.rW  c                     X S3-  [        5       -  nUR                  SS9  [        USSS9n[        R                  " U5      n[        XC5        [        R                  " U5      n[        XC5        g )NdataTparentsr~   rC   rK  rM  rN  s        r&   /test_ignore_no_private_directories_in_base_pathr`  d  sm    
 ,d++df4GMM$M+GB78:E &G') (G')r(   c           	         S/S-  S/S-  -   n[         R                  " [         R                  " [        [	        U5      5      5      [         R                  " U5      R                  5       /SS/S9n[        R                  " U[        U 5      S/S9  U S-  nUR                  5         [        R                  " U[        U5      S/S9  [        R                  " U S	/S
9nUR                  U5      (       d   eg )Nxxxr   yyyrl   _partr  partition_cols_private_duplicate_private)ignore_prefixes)r   r   rS   r   r   dictionary_encoder   write_to_datasetr   r   r   r    )r"   partr   private_duplicater3   s        r&   test_ignore_custom_prefixesro  w  s    7Q;%1$DHH
s4y!"
((* w!E
 s7|WIF"66s#45(/y2 ==*/D ;;ur(   c                     U S-  nUR                  5         [        R                  " U5      nUR                  5       nUR                  S:X  d   eUR
                  S:X  d   eg )Nr6   r   )r   r   r2   r3   r]   r  )r"   	empty_dirr6   r%   s       r&   test_empty_directoryrr    sW    )#IOO	*G\\^F??a"""r(   c                    SS K nSS KJn  SS KJn  UR
                  " [        S5      [        S5      [        [        S5      5      [        R                  /S-  [        R                  " SSSS9R                  S	5      S
.5      nUR                  R                  5       nSS/n	[        R                  R!                  XrSSS9n
UR"                  " XU	US9  [$        R&                  R)                  [+        U 5      S5      nUb9  UR-                  US5       nUR.                  " U
R0                  U5        S S S 5        OUR.                  " U
R0                  U5        UR2                  " U US9n[5        UR0                  R6                  5      nU[5        U
R0                  R6                  5      :X  d   eUR9                  5       nUR;                  5       nUR                  R                  5       nU	US[=        U	5      -  S  :X  d   eUU   nU	 H  nUU   R                  S5      UU'   M     U(       a@  UR?                  S5      R@                  RC                  5       nUS   R                  U5      US'   URD                  " UU5        g ! , (       d  f       GN5= f)Nr   
aaabbbbccc
eefeffgeeer~   
2017-01-01
2017-01-11datetime64[D]r@   datetime64[ns])group1group2numnanr   rz  r{  F)r  safepreserve_indexr   _common_metadatarQ  r   r   )#pandaspandas.testingtestingpyarrow.parquetparquetrQ   listr   rR   r}  rW   r   r1   tolistr   r   r   rl  osr$   r   r   rS  write_metadatar  r2   r   r  r3   rY   r   r  r   to_pandas_dtyper   )r5   r   r  
index_namerP   r   r   	output_dfcolspartition_byoutput_tablemetadata_pathr   r6   dataset_colsinput_tableinput_dfinput_df_colscolexpected_date_types                       r&   &_test_write_to_dataset_with_partitionsr    s?      |$|$E"Ix"}		,OLSS I ##%Dh'L88''	u7< ( >L#-/ GGLLY1CDM__]D1Ql1115 21 	,--}=	+57G w~~++,L3|22889999,,.K$$&H $$++-M=c,.?)?)@AAAA~H"3..z:	#  #\\&166FFH%f-445GH	&)X.= 21s   I99
Jc                    SS K nSS KJn  UR                  " [	        S5      [	        S5      [	        [        S5      5      [        R                  " SSSS9R                  S	5      S
.5      nUR                  R                  5       n[        R                  R                  U5      nUc  [        5       nO)[        U[         5      (       d  [#        [%        U5      5      nSn[        U5       H  nUR&                  " X`US9  M     [)        [+        U 5      SSS9n	UR-                  U	5      n
U
 Vs/ s H&  oR.                  R1                  S5      (       d  M$  UPM(     nn[3        U5      U:X  d   eUR4                  " XS9R7                  5       nUR9                  5       nUR;                  5       nX   n[<        R>                  " XN5        g s  snf )Nr   rt  ru  r~   rv  rw  rx  r@   ry  )rz  r{  r|  r   rC   r   FT)allow_not_found	recursiver  ) r  r  r  rQ   r  r   rR   rW   r   r1   r  r   r   r   r   r   r   r   r   rl  r   r   r   r$   endswithr   r2   r3   rY   drop_duplicatesr   r   )r5   r   rP   r   r  r  r  nr   selectorinfosinfooutput_filesr  r  s                  r&   $_test_write_to_dataset_no_partitionsr    s     |$|$E"I		,OLSS	 I ##%D88''	2L$&

J//!-
";<
 	
A1X
L'1	3  C	NE&*,H $$X.E%*MUTii.@.@.LDULM|!!! ##
df  $$&H'')H~H). Ns   '#G G c                 ,    [        [        U 5      5        g r*   r  r   r"   s    r&   %test_write_to_dataset_with_partitionsr    s    *3w<8r(   c                    [         R                  " [         R                  " S[         R                  " 5       S9[         R                  " S[         R                  " 5       S9[         R                  " S[         R                  " 5       S9[         R                  " S[         R
                  " 5       S9[         R                  " S[         R                  " SS9S9/5      n[        [        U 5      US	9  g )
Nrz  )r   r{  r|  r}  r   us)unitr  )	r   r  r  r=   r  int32	timestampr  r   )r"   r  s     r&   0test_write_to_dataset_with_partitions_and_schemar    s    YY		<		<RXXZ8RXXZ8bll.EF	H IF
 +GV%r(   c                 *    [        [        U 5      SS9  g )Nr  )r  r  r  s    r&   4test_write_to_dataset_with_partitions_and_index_namer    s    *G/r(   c                 ,    [        [        U 5      5        g r*   )r  r   r  s    r&   #test_write_to_dataset_no_partitionsr    s    (W6r(   c                 <    [        U S-  5        [        U S-  5        g )Ntest1test2)r  r  r  s    r&   test_write_to_dataset_pathlibr    s    *7W+<=(7):;r(   c                    Uu  p#[         R                  " [        SS9   [        U S-  US9  S S S 5        [         R                  " [        SS9   [	        U S-  US9  S S S 5        g ! , (       d  f       N>= f! , (       d  f       g = f)Nz"path-like objects are only allowedr   r  r   r  )r   rz   r   r  r  )r"   r   r   rH  s       r&   &test_write_to_dataset_pathlib_nonlocalr  #  sr     EB	y(L	M.g"	. 
N 
y(L	M,g"	. 
N	M	 
N	M 
N	Ms   A$A5$
A25
Bwin32z,test fails because of unsupported characters)r{   c                     U u  p[        X!S9  g Nr   )r  r   s      r&   *test_write_to_dataset_with_partitions_s3fsr  2  s     HB*r(   c                     U u  p[        X!S9  g r  )r  r   s      r&   (test_write_to_dataset_no_partitions_s3fsr  >  s     HB(r(   c                 "   [         R                  " S/ SQ05      n[        R                  R	                  U5      n[        U 5      n[        R                  " X#[        5       S9  [        R                  " U5      nUR                  U5      (       d   eg )Nr  r   r   )rP   rQ   r   r   r   r   r   rl  r   r   r    )r"   rb   r   r$   r%   s        r&    test_write_to_dataset_filesystemr  G  sh    	sI&	'BHH  $Ew<D0AB]]4 F==r(   c                    U S-  n[        5       n[        R                  " [        R                  " U5      [        R
                  R                  U5      S.SS/S9n[        R                  R                  U5      nSn[        R                  " X%R                  5       n[        U5       H  nUR                  U5        M     S S S 5        [        R                  " U5      n	U	R                   R"                  U:X  d   eU S-  n
UR%                  ['        U
5      5       n[        R(                  " UR                  U5        S S S 5        [        R*                  " XS9nU$ ! , (       d  f       N= f! , (       d  f       N5= f)	Nr   )rl   r/   rl   r/   r0   r   	_metadatar   )r   rP   rQ   rR   rW   r   r   r   r   r   r   ParquetWriterr  r   r   ParquetFiler  num_row_groupsr   r   r  r2   )r"   rv   r$   r,   rb   r   
num_groupswriterr   readerr  r   r6   s                r&   _make_dataset_for_picklingr  R  s+   ^#DE	1))//!$ "
$B HH  $EJ			$	-z"Au% # 
. ^^D!F??))Z777k)M		!	!#m"4	5
%,,* 
6 #G N 
.	- 
6	5s   $E"E*
E'*
E8c                 F   ^ U4S jn[        U 5      nU" U5      (       d   eg )Nc                 J   > U TR                  TR                  U 5      5      :H  $ r*   )loadsdumps)objpickle_modules    r&   is_pickleable*test_pickle_dataset.<locals>.is_pickleablep  s$    m))-*=*=c*BCCCr(   )r  )r"   r  r  r6   s    `  r&   test_pickle_datasetr  n  s&    D )1G!!!!r(   c                 H   U S-  n[         R                  " / SQ/ SQ/ SQS.5      n[        R                  R	                  U5      n[
        R                  " U[        U5      SS/S9  [
        R                  " U5      R                  5       n[
        R                  " X1S	-  5        g )
Nz
ARROW-3208)r  r~   g      @d     r   g333333=@)r  r~   r   r  r  r   r   )r   r   r   r   r   r   r   )onetwothreer  r  )	root_pathrg  zoutput.parquet)rP   rQ   r   r   r   r   rl  r   r2   r3   r   )r"   r$   rb   r   s       r&   test_partitioned_datasetr  w  s     \!D	0,& 
B
 HH  $ET(-u~7d#((*ENN5!112r(   c                    U S-  n[         R                  " [        S5       Vs/ s H  n[        R                  " S5      PM     snS-  /S/S9n[         R                  " [        S5       Vs/ s H  n[        R                  " S5      PM     snS-  /S/S9n[
        R                  " U[        U5      S9  [
        R                  " U[        U5      S9  [
        R                  " US/S9R                  5       nUS   R                  S5      R                  5       US   R                  S5      R                  5       /nUS   R                  S	:X  d   eUS   R                  S5      US   R                  S
5      pUR                  US   5      (       a  UR                  US
   5      (       d   eg UR                  US
   5      (       d   eUR                  US   5      (       d   eg s  snf s  snf )NzARROW-3325-datasetrC   r~   f0re  )r  )read_dictionaryr   r   r   )r   r   r   r	   randsr   rl  r   r2   r3   chunkrk  
num_chunksr    )	r"   r$   r   t1t2r%   	ex_chunksc0c1s	            r&   test_dataset_read_dictionaryr    s   ))D	E!H5HqDJJrNH5:;D6	JB	E!H5HqDJJrNH5:;D6	JBc$i0c$i0tf&&*df  AQ113AQ1135I !91$$$AY__Q!3	yy1yy1&&&&yy1&&&&yy1&&&&% 65s    G"+ G'c                    [         R                  " S[         R                  " / SQ[         R                  " 5       5      05      n[        R
                  " XS-  5        [        R
                  " XS-  5        [         R                  " S/5      n[        R                  " U S-  US9n[         R                  " S/ SQ0US9nUR                  U5      (       d   e[        R                  " XS9n[         R                  " S/ SQ0US9nUR                  U5      (       d   e[        R                  " XS9n[         R                  " S/ SQ0US9nUR                  5       R                  U5      (       d   eg )Nr   r   zdata1.parquetzdata2.parquet)r   r  r  )r   r   r   r   r   r   )r   r   rS   r  r   r   r  r   r    r2   r3   )r"   r   r  r%   r   s        r&   test_read_table_schemar    s   HHc288Irxxz:;<ENN5O34NN5O34YY'(F ]]7_4VDFxxi(8H=="""" ]]72Fxx01&AH==""""w6Fxx01&AH;;=))))r(   c                    [         R                  " [         R                  " / SQ[         R                  " 5       5      [         R                  " / SQ[         R                  " 5       5      S.5      n[
        R                  " XS-  5        [
        R                  " U S-  SS/S9n[         R                  " SS/5      nUR                  SS/:X  d   eUR                  U:X  d   eg )Nr   r   r   r   r0   )r   r  )
r   r   rS   r  r,  r   r   r   r  r4   )r"   r   r%   expected_schemas       r&   *test_read_table_duplicate_column_selectionr    s    HH288Irxxz:88Irxxz:< =ENN5N23]]7^3c3ZHFii @AO3*,,,==O+++r(   c                    SS K Jn  U S-  nUS-  S-  S-  R                  SS9  [        R                  " S/ S	Q05      n[
        R                  " U[        US-  S-  S-  S
-  5      5        UR                  / SQS9n[
        R                  " [        U5      US9nUR                  / SQ:X  d   e[
        R                  " [        U5      US9R                  5       nUR                  / SQ:X  d   eg )Nr   test_partitioning20121001Tr^  r   r   r   )yearmonthday)field_names)partitioning)r   r  r  r  )pyarrow.datasetr6   r   r   r   r   r   r   r  r   r4   r2   r3   )r"   dsr  r   rm  r%   s         r&   test_dataset_partitioningr    s      --I$%,,T,:HHc9%&ENNs9v%,t3nDEG ??'??@D]]IT+F"????IT++/46 "????r(   c                    [         R                  " S/ SQ05      n[        R                  " XS-  5        [	        [        U 5      [        5       5      n[        R                  " SUS9nUR                  5       nUR                  U5      (       d   eg )Nr   r   r   rI  r   )
r   r   r   r   r   r   r   r2   r3   r    )r"   r   r   r6   r%   s        r&   #test_parquet_dataset_new_filesystemr    sk    HHc9%&ENN5N23"3w<1BCJ
;G\\^F==r(   c                 ^   [         R                  " S5      nUR                  S5      n[        R                  " S/ SQ05      n[
        R                  " X0S-  5        [        U 5      R                  SS5      n[
        R                  " XBS9nUS	-   nUR                  S
   R                  U:X  d   eg )Nfsspecfiler   r   r   \r   r   z/data.parquetr   )r   importorskipr   r   r   r   r   r   replacer2   	fragmentsr$   )r"   r  r   r   r$   r6   r   s          r&   6test_parquet_dataset_partitions_piece_path_with_fsspecr    s       *F""6*JHHc9%&ENN5N23 w<c*D%G o%HQ$$000r(   c                 
  ^ [         R                  " S/ SQ05      nU S-  n/ mU4S jnSn[        R                  " XS/UUS9  US-  S-  US	-  S-  US
-  S-  1n[	        [        [        R                  T5      5      nXe:X  d   eg )Nr   r   r  c                 <   > TR                  U R                  5        g r*   )r   r$   )written_filepaths_writtens    r&   file_visitorDtest_parquet_write_to_dataset_exposed_keywords.<locals>.file_visitor  s    \../r(   zpart-{i}.parquet)r  r  basename_template1zpart-0.parquet23)r   r   r   rl  r   rs   pathlibPath)r"   r   r$   r  r	  expected_pathspaths_written_setr  s          @r&   .test_parquet_write_to_dataset_exposed_keywordsr    s    HHc9%&E^#DM0 +3%%1*;=
 	s
%%s
%%s
%%N
 Cm<=...r(   write_dataset_kwarg))r   T)r   Fc                    SSK Jn  [        R                  " S/ SQ05      nU S-  n[        R
                  " UR                  5      nUu  pgU[        R
                  " [        R                  5      R                  ;  d   eXeR                  ;   d   e[        R                  R                  USSS9 n[        R                  " X440 Xg0D6  UR                  S   u  pnX   U:X  d   e SSS5        g! , (       d  f       g= f)	zEVerify kwargs in pq.write_to_dataset are passed onto ds.write_datasetr   Nr   r   zout.parquetwrite_datasetT)autospec)r  r6   r   r   inspect	signaturer  r   rl  
parametersmockpatchrV   
mock_calls)r"   r  r  r   r$   r  keyargmock_write_dataset_name_argsr   s               r&   #test_write_to_dataset_kwargs_passedr!    s     !HHc9%&E]"D!!""2"23I"HC g''(;(;<GGGGG&&&&&			2		>!
E6C:61<<Q?f{c!!!	 
?	>	>s   .5C--
C;c                    [         R                  " [         R                  " / SQ/ SQS9/ SQS.5      n[        R                  " U5      nU S-  n[
        R                  " X S-  S/S9  UR                  5        Vs/ s H%  oDR                  5       (       d  M  UR                  PM'     nn[        U5      S	:X  d   eS
U;  d   eg s  snf )N)r   r:   r   r9   r   r   )catr  r6   r#  rf  r   zcat=c)rP   rQ   r   r   r   r   rl  iterdiris_dirr   r   )r"   rb   r   r$   r   subdirss         r&   'test_write_to_dataset_category_observedr'  '  s    
 
~~o/J 
B HHRLEYD"E7  $||~<~!vqvv~G<w<1'!!! =s   9CC)r~   rC   )NNNr*   )r  )nr   r  r  r  sysnumpyrR   ImportErrorr   unittest.mockr  pyarrowr   pyarrow.computecomputepcr   r   r   r   r   r   r   pyarrow.testsr	   pyarrow.utilr
   r  r  r   pyarrow.tests.parquet.commonr   r   r   r   r  rP   r  r  r   markr6   
pytestmarkr'   r-   r7   rf   ry   xfailr   AssertionErrorr   r   r   r   r   r   parametrizer  castr  r   r   s3r   r   r   r   r+   rX   r   r
  r*  r0  r7  r;  r?  rB  rF  rO  rV  r[  r`  ro  rr  r  r  r  r  r  r  r  r  skipifplatformr  r  r  r  r  r  r  r  r  r  r  r  r  r!  r'   r(   r&   <module>r=     sD  $   	  
    H H   A A kk!!6;;#6#67
 & 3 3
 	- 	- C, C,L ! !B  ~&B	   '1 '1T = =  ! !B +4 +4\ %, %,P N N. /0012((:.2((8S1A5((8S166rxxzBQF	HI (EF GI 4 A A.             -  -
$2N+"\/  C C* 5$ 5$N "N "NJ ( ( , ,$ $ $ 	,
 Sz2* 3 * * *$ * *$ Sz2* 3 *"*# 7;266::/| 59+/\ 9 9 % % / /
 7 7 < <
 
.  
. CLLG+I  KK  
       8 " " 3 3 '.*.
,@0 1"/0 . 1 "	"* " "](  	B   	B  NBs5   Z/ Z> 
[ /Z;:Z;>[
	[

[[