
    <>i'G                        S SK r S SKrS SKJr   S SKJr  S SKJrJ	r	J
r
Jr   S SKrS SKJr  \ R$                  R
                  r\ R$                  R                  S 5       rS rS r\ R$                  R                  S 5       r\ R$                  R                  S 5       r\ R$                  R                  \ R$                  R3                  S	S\R4                  " 5       /5      S
 5       5       r\ R$                  R8                  \ R$                  R                  S 5       5       r\ R$                  R                  \ R$                  R3                  S	S\R4                  " 5       /5      S 5       5       r\ R$                  R                  \ R$                  R>                  S 5       5       r \ R$                  R                  \ R$                  R>                  S 5       5       r!\ R$                  R                  \ R$                  R>                  S 5       5       r"\ R$                  RF                  S 5       r$S r%S r&S r'S r(\ R$                  R3                  S\RR                  S4\RR                  S4\RT                  S4\RT                  S4/5      \ R$                  R3                  SSS/5      S 5       5       r+\ R$                  R3                  S/ SQ5      S  5       r,S! r-g! \ a    Sr GNf = f! \ a    S=rr GNf = f)"    N)fs)_read_table_test_dataframe_test_table_range_integersc                    [        S5      nSUS'   [        R                  R                  USS9n[        R                  " 5       n[
        R                  " X2R                  SS9n/ n[        S5       HU  nXaS'   [        R                  R                  USS9nUR                  U5        UR                  UR                  5       5        MW     UR                  5         UR                  5       n[        [        R                  " U5      5      n[         R"                  " US	S
9n	[$        R&                  " UR)                  5       U	5        g Nd   r   	unique_idFpreserve_index2.6version
   Tignore_index)r   paTablefrom_pandasBufferOutputStreampqParquetWriterschemarangewrite_tableappendcopyclosegetvaluer   BufferReaderpdconcattmassert_frame_equal	to_pandas
tempdirdfarrow_tableoutwriterframesibufresultexpecteds
             h/var/www/html/land-tabula/venv/lib/python3.13/site-packages/pyarrow/tests/parquet/test_parquet_writer.py#test_parquet_incremental_file_buildr3   ,   s    		BB{O((&&r%&@K



!Cc#5#5uEFF2Y;hh**2e*D;'bggi   LLN
,,.C-.Fyyd3H&**,h7    c                 n   [         R                  " S[         R                  " 5       5      [         R                  " S[         R                  " 5       5      /n[         R                  " U5      n[         R
                  " S/5      [         R
                  " S/5      /n[         R                  R                  USS/5      nU S-  n[        R                  " XRSSSS	9 n[        R                  " [        5         UR                  U5        S S S 5        S S S 5        g ! , (       d  f       N= f! , (       d  f       g = f)
NPOSdesc   blazsimple_validate_schema.parquetr   snappyspark)r   compressionflavor)r   fielduint32stringr   arrayr   from_arraysr   r   pytestraises
ValueErrorr   )r(   simple_fieldssimple_schemasimple_from_arraysimple_tablepathws          r2    test_validate_schema_write_tablerL   G   s     			$
%M
 IIm,M 1#%(9:88''(9E6?KL55D			$"'&.w
@CD]]:&MM,' '
@ 
@ '&
@ 
@s$   D&2DD&
D#	D&&
D4c                    [         R                  " [        5         [        R                  " [        R
                  " S[        R                  " 5       5      /5      n[        R                  " S U5        S S S 5        [         R                  " [        5         [        R                  " U S-  S 5        S S S 5        g ! , (       d  f       NL= f! , (       d  f       g = f)Nx	some_path)	rC   rD   	TypeErrorr   r   r>   int32r   r   )r(   some_schemas     r2   test_parquet_invalid_writerrS   ]   s    	y	!ii#rxxz!: ;<
{+ 
" 
y	!
;.5 
"	!	 
"	! 
"	!s   AB8C	8
C	
Cc                    [        S5      nSUS'   [        R                  R                  USS9n[        R                  " 5       n[
        R                  " X2R                  SS9 n/ n[        S5       HU  nXaS'   [        R                  R                  USS9nUR                  U5        UR                  UR                  5       5        MW     S S S 5        UR                  5       n[        [        R                  " U5      5      n[        R                   " WS	S
9n	["        R$                  " UR'                  5       U	5        g ! , (       d  f       Nx= fr	   )r   r   r   r   r   r   r   r   r   r   r   r   r    r   r!   r"   r#   r$   r%   r&   r'   s
             r2   test_parquet_writer_context_objrU   g   s    		BB{O((&&r%&@K



!C			#115	AVrA{O((..r%.HK{+MM"'')$  
B ,,.C-.Fyyd3H&**,h7 
B	As   #A'D<<
E
c                    [        S5      nSUS'   [        R                  R                  USS9n[        R                  " 5       nSn [
        R                  " UUR                  SS9 n/ n[        S	5       Hf  nXqS'   [        R                  R                  USS9nUR                  U5        UR                  UR                  5       5        US
:X  d  M]  [        U5      e   S S S 5        UR                  5       n	[!        [        R"                  " U	5      5      n
[$        R&                  " WSS9n[(        R*                  " U
R-                  5       U5        g ! , (       d  f       Nx= f! [         a  n[        U5      U:X  d   e S nANS nAff = f)Nr
   r   r   Fr   zArtificial Errorr   r   r      Tr   )r   r   r   r   r   r   r   r   r   r   r   r   rE   	Exceptionstrr    r   r!   r"   r#   r$   r%   r&   )r(   r)   r*   r+   
error_textr,   r-   r.   er/   r0   r1   s               r2   .test_parquet_writer_context_obj_with_exceptionr\      sF   		BB{O((&&r%&@K



!C#J$c)00&+-06F2Y"#; hh222e2L"";/bggi(6$Z00 - ,,.C-.Fyyd3H&**,h7'- -  $1v###$s=    E" 'A'EEE" 
EE" E" "
F,FF
filesystemc                    [        S5      n[        R                  R                  USS9n[        R                  R                  USS9n[        U S-  5      n[        U S-  5      n[        R                  " XSR                  USS9 nUR                  U5        S S S 5        [        U5      R                  5       n[        R                  " X5        [        R                  " XcR                  USS9 nUR                  U5        S S S 5        [        U5      R                  5       n[        R                  " X5        [        R                  " XSR                  USS9 nUR                  U5        S S S 5        [        U5      R                  5       n[        R                  " X5        [        R                  " XcR                  USS9 nUR                  U5        S S S 5        [        U5      R                  5       n[        R                  " X5        g ! , (       d  f       GNz= f! , (       d  f       GN#= f! , (       d  f       N= f! , (       d  f       Ns= f)Nr
   Fr   zdata_table.parquetzdata_batch.parquetr   r]   r   )r   r   r   r   RecordBatchrY   r   r   r   r   r   r&   r$   r%   write_batchwrite)	r(   r]   r)   tablebatch
path_table
path_batchr,   r0   s	            r2   "test_parquet_writer_write_wrappersrg      s    
	BHH  E :ENN&&r%&@EW334JW334J			LLZ
	5!

 $..0F&%			LLZ
	5!

 $..0F&%			LLZ
	U

 $..0F&%			LLZ
	U

 $..0F&%=
 

 

 

 
s0   H	-HH-?H>	
H
H*-
H;>
Ic                    ^ ^^ SmSmUUU 4S jnU" TS-  TS-
  S5        U" TS-  TS5        U" TS-  TS-   S5        U" TS-   TS-   S5        U" TS-  TS-  S5        U" TS S5        U" TS-   S S5        g )Ni   i   c                 b  > [         R                  R                  [        U S5      /S/S9nUc  [        R
                  " UTS-  5        O[        R
                  " UTS-  US9  [        R                  " TS-  5      nUc  T
OUnUR                  U:X  d   e[        UT	5      n[        US-
  5       H$  nUR                  U5      R                  U:X  a  M$   e   XUS-
  -  -
  nUS:X  a%  UR                  US-
  5      R                  U:X  d   eg UR                  US-
  5      R                  U:X  d   eg )NbrN   )namesztest.parquet)row_group_sizer8   r   )r   r   rB   r   r   r   read_metadatanum_row_groupsminr   	row_groupnum_rows)	data_size
chunk_sizeexpect_num_chunksrc   metadataexpected_chunk_sizelatched_chunk_size	chunk_idx	remainderabs_max_chunk_sizedefault_chunk_sizer(   s            r2   check_chunk_size8test_parquet_writer_chunk_size.<locals>.check_chunk_size   sO   $$Is+&
 %  NN5'N":;NN5'N"::V##Gn$<=4>4F0J&&*;;;; !46HI0145I%%i099=OOOO 6 8IA8M!NO	>%%!A%''/x3EF F F %%&7!&;<EERRRr4      r
      r8    )r(   r|   rz   r{   s   ` @@r2   test_parquet_writer_chunk_sizer      s     %)S. '!+-?#-EqI'!+-?C'!+-?#-EqI'#-/AC/GK '!+-?!-CQG 'q1'!+T15r4   c                 f   [        S5      n[        R                  R                  USS9n[	        U S-  5      n[
        R                  " XCR                  USS9 nUR                  U5        S S S 5        [        U5      R                  5       n[        R                  " Xb5        g ! , (       d  f       N>= f)Nr
   Fr   zdata.parquetr   r_   )r   r   r   r   rY   r   r   r   r   r   r&   r$   r%   )r(   r]   r)   rc   rJ   r,   r0   s          r2   $test_parquet_writer_filesystem_localr      s     
	BHH  E :Ew'(D			llz5
	5!

 ((*F&%
 
s   B""
B0c                 T   [        S5      n[        R                  R                  USS9nU u  p4n[        R
                  " XRR                  USS9 nUR                  U5        S S S 5        [        U5      R                  5       n[        R                  " Xq5        g ! , (       d  f       N>= f)Nr
   Fr   r   r_   r   r   r   r   r   r   r   r   r   r&   r$   r%   s3_example_fsr)   rc   r   urirJ   r,   r0   s           r2   !test_parquet_writer_filesystem_s3r     s     
	BHH  E :E!MBT			llr5
	5!

 '')F&%
 
s   B
B'c                 N   [        S5      n[        R                  R                  USS9nU u  p4n[        R
                  " XBR                  SS9 nUR                  U5        S S S 5        [        XSS9R                  5       n[        R                  " Xq5        g ! , (       d  f       N<= f)Nr
   Fr   r   r   r]   r   r   s           r2   %test_parquet_writer_filesystem_s3_urir     s     
	BHH  E :E!MBT			#||U	;v5! 
< -779F&%	 
<	;s   B
B$c                 X   [        S5      n[        R                  R                  USS9nU u  p4US-   n[        R
                  " XRR                  USS9 nUR                  U5        S S S 5        [        XSS9R                  5       n[        R                  " Xq5        g ! , (       d  f       N<= f)Nr
   Fr   z/test.parquetr   r_   r   r   )s3_example_s3fsr)   rc   r   	directoryrJ   r,   r0   s           r2   #test_parquet_writer_filesystem_s3fsr   +  s     
	BHH  E :E#MB&D			llr5
	5!

 -779F&%
 
s   B
B)c                     [        S5      n [        R                  " 5       n[        R                  " [
        SS9   [        R                  " [        R                  " 5       U R                  US9  S S S 5        g ! , (       d  f       g = f)Nr
   zspecified path is file-likematchr   )r   r   LocalFileSystemrC   rD   rE   r   r   r   r   r   )rc   r]   s     r2   ,test_parquet_writer_filesystem_buffer_raisesr   =  sZ    E##%J 
z)F	G
!!#U\\j	
 
H	G	Gs   4A77
Bc                 <   [         R                  " S/ SQ05      nU S-  n[        R                  " X!R                  5       nUR                  U5        S S S 5        [        R                  " U5      nSUR                  ;   d   eUR                  S   (       d   eU S-  n[        R                  " XQR                  SS9 nUR                  U5        S S S 5        [        R                  " U5      nUR                  b   eg ! , (       d  f       N= f! , (       d  f       NE= f)Na)r8   r~   r   ztest_with_schema.parquets   ARROW:schemaztest_without_schema.parquetF)store_schema)r   rc   r   r   r   r   rm   ru   )r(   rc   path1r,   metapath2s         r2    test_parquet_writer_store_schemar   I  s    HHc9%&E 00E			%	.&5! 
/ E"Ddmm+++==))) 33E			%E	Bf5! 
C E"D==    
/	. 
C	Bs    C<<D<
D

Dc                    [         R                  R                  [         R                  " / SS9/S/5      nU S-  n[        R
                  " X!R                  5       nUR                  U5        UR                  SSS.5        UR                  SS	S
.5        S S S 5        [        R                  " U5      nUR                  R                  nUS   S:X  d   eUS   S:X  d   eUS   S:X  d   eg ! , (       d  f       N\= f)NrQ   typef0zmetadata.parquet1rN   )key1key223)r   key3s   key1   1s   key2   2s   key3   3)r   r   rB   rA   r   r   r   r   add_key_value_metadataParquetFileru   )r(   rc   rJ   r,   readerru   s         r2   -test_parquet_writer_append_key_value_metadatar   ^  s    HH  "((2G"<!=vFE''D			$	-5!%%sC&@A%%sC&@A 
. ^^D!F''HG$$$G$$$G$$$ 
.	-s   :C++
C9c           	         [         R                  " S[        S5      05      n[        R                  " XS-  SSS9  [        R                  " XS-  SSSS	9  [        R                  " XS
-  SSSSS.S	9  [        R
                  " U S-  5      n[        R
                  " U S-  5      n[        R
                  " U S
-  5      nUR                  U5      (       d   eUR                  U5      (       d   e[        R                  " U S-  5      n[        R                  " U S-  5      n[        R                  " U S
-  5      nUR                  UR                  :X  d   eUR                  UR                  :X  d   e[        UR                  5       H  nUR                  U5      n	UR                  U5      n
UR                  U5      nU	R                  U
R                  :X  d   eU	R                  UR                  :X  d   eU	R                  U
R                  :  d   eU	R                  UR                  :  d   eU
R                  UR                  :  a  M   e   g )Nr   i zunchunked.parquetFPLAIN)use_dictionarycolumn_encodingzchunked-default.parquetT)r   r   use_content_defined_chunkingzchunked-custom.parquet      min_chunk_sizemax_chunk_size)r   rc   r   r   r   
read_tableequalsrm   rn   rp   rq   total_byte_size)r(   rc   	unchunkedchunked_defaultchunked_customunchunked_metadatachunked_default_metadatachunked_custom_metadatar.   rg_unchunkedrg_chunked_defaultrg_chunked_customs               r2   %test_parquet_content_defined_chunkingr   m  s7   HHc5>*+E NN5$77"'#*, NN5$=="'#*046 NN5$<<"'#*CICI1KL g(;;<ImmG.G$GHO]]7-E#EFNO,,,,N++++ ))'4G*GH!//:S0ST ..w9Q/QR,,0H0W0WWWW,,0G0V0VVVV%445)33A65??B3==a@$$(:(C(CCCC$$(9(B(BBBB ++.@.P.PPPP++.?.O.OOOO!114E4U4UUUU 6r4   c                 N   [         R                  " S[        S5      05      nU S-  nSn[        R                  " [
        US9   SSS.n[        R                  " XUS	9  S S S 5        SS
S.S4SS0S4SS0S4/nU H<  u  pC[        R                  " [        US9   [        R                  " XUS	9  S S S 5        M>     [        R                  " XS
S	9  SSS.n[        R                  " XUS	9  SSSS.n[        R                  " XUS	9  g ! , (       d  f       N= f! , (       d  f       M  = f)Nr   r
   zchunked-invalid.parquetz2max_chunk_size must be greater than min_chunk_sizer   r   r   r   )r   T)r   unknown_optionzEUnknown options in 'use_content_defined_chunking': {'unknown_option'}r   zEMissing options in 'use_content_defined_chunking': {'max_chunk_size'}r   zEMissing options in 'use_content_defined_chunking': {'min_chunk_size'}r8   )r   r   
norm_level)	r   rc   r   rC   rD   rX   r   r   rE   )r(   rc   rJ   msgcdc_optionscasess         r2   0test_parquet_content_defined_chunking_parametersr     s   HHc5:&'E..D ?C	y	,)/6J
uM 
-  )DAS	

 y)S	

 y)S	
E "]]:S1NN5[Q 21 "
 NN5TB &,vFKNN5[I &,vUVWKNN5[I? 
-	,& 21s   DD
D
D$	ztime_type, time_unitsmsusnsutc_flag_valFTc           	         U S-  n/ SQn[         R                  " S[         R                  " XR" U5      S905      n[         R                  " SU" U5      4/5      n[        R
                  " UUUS9 nUR                  U5        S S S 5        [        R                  " XGS9n	U	R                  SS9  U	R                  U5      (       d   eg ! , (       d  f       NJ= f)	Nztime_adjusted_to_utc.parquet)r   {   '  iQ time_colr   )wherer   write_time_adjusted_to_utcr   Tfull)
r   rc   rA   r   r   r   r   r   validater   )
r(   r   	time_type	time_unitfilenametime_valuesrc   r   r,   r0   s
             r2   ,test_arrow_writer_props_time_adjusted_to_utcr     s     77H*KHHBHH[y/CD E YY	Yy)* F 
		#/
 
5!
 ]]83F
OOO==
 
s   /C
Cmax_rows_per_page)r8   r   r
     Nc                 \   U S-  n[         R                  " [         R                  " / SQ[         R                  " 5       S9[         R                  " / SQ[         R                  " 5       S9S.5      n[         R
                  " S[         R                  " 5       4S[         R                  " 5       4/5      n[        R                  " UUUS9 nUR                  U5        S S S 5        [        R                  " X$S	9nUR                  S
S9  UR                  U5      (       d   eg ! , (       d  f       NJ= f)Nzmax_rows_per_page.parquet)r8   r~   r      rW         r   )g      &@g      (@g      *@g      ,@g      .@g      0@g      1@)rN   yrN   r   r   r   r   r   Tr   )r   rc   rA   int8float16r   r   r   r   r   r   r   )r(   r   r   rc   r   r,   r0   s          r2   #test_writer_props_max_rows_per_pager     s     44HHHXX+"'')<XX@rzz|T E
 YY	bggi	bjjl F
 
		+
 
5!
 ]]83F
OOO==
 
s   D
D+c                    [         R                  " S[         R                  " [        S5      5      05      n[        R
                  " 5       n/ nS He  nU  SU S3n[        R                  " UUR                  US9 nUR                  U5        S S S 5        UR                  UR                  U5      5        Mg     US   R                  US   R                  :  d   eg ! , (       d  f       NU= f)	NrN   i@B )r   r   z/max_rows_per_page_z.parquetr   r   r8   )r   rc   rA   r   r   r   r   r   r   r   r   get_file_infosize)r(   rc   local
file_infosmax_rowsrJ   r,   s          r2   -test_writer_props_max_rows_per_page_file_sizer     s    HHRXXeI&' E  EJ#-hZx@<<&
 u%
 	%--d34 $ a=
1 2 2222
 
s   :C
C)	).rC   pyarrowr   r   pyarrow.parquetparquetr   pyarrow.tests.parquet.commonr   r   r   r   ImportErrorpandasr"   pandas.testingtestingr$   mark
pytestmarkr3   rL   rS   rU   r\   parametrizer   rg   large_memoryr   r   s3r   r   r   numpyr   r   r   r   r   time32time64r   r   r   r   r4   r2   <module>r     s  $    L L [[  
 8 84(,6 8 80 8 8> ( %&	 
%&P &6  &6R ( &	 
& &  & 
&  
& &  &  
 
!*%+V\&JR /YYYYYYYY	2  %7  8 B  	 83c  	B  NBs"   K 
K KK
K-,K-