
    <>igF                        S SK r S SKJr  S SKJr  S SKrS SKJr  S SKr	S SK
r
Sr S SKJr  S SKJr   S SKJr  S SKJs  Jr  \
R0                  R                   rSrSrS	rS
r\\\\0rSrSr / SQr!\SS/0r"S r#\\"4S jr$S r%\4S jr&S r'\S 5       r(S(S jr)\
R0                  RU                  \SS9S 5       r+\
R0                  RU                  \SS9\
R0                  RY                  SSS/5      S 5       5       r-\
R0                  RU                  \SS9\
R0                  RY                  S/ SQ5      S 5       5       r.\
R0                  RU                  \SS9\
R0                  RY                  S/ S Q5      S! 5       5       r/\
R0                  RU                  \SS9\
R0                  RY                  S/ S"Q5      S# 5       5       r0\
R0                  RU                  \(       + S$S9S% 5       r1\
R0                  RU                  \SS9S& 5       r2\
R0                  RU                  \SS9S' 5       r3g! \ a    SrSr GNf = f! \ a    Sr GNf = f))    N)contextmanager)	timedeltaFInMemoryKmsClientTs   0123456789112345
footer_keys   1234567890123450col_keys   2345678901234501col2_keyyearn_legsanimalr   r   c                  B    [         R                  " / SQ/ SQ/ SQS.5      $ )N)i      r     r   )   r      r      d   )FlamingoParrotDogHorsezBrittle stars	Centipeder
   )patable     d/var/www/html/land-tabula/venv/lib/python3.13/site-packages/pyarrow/tests/test_dataset_encryption.pycreate_sample_tabler    :   s#    888*	
 r   c           
      B    [         R                  " U SUS[        SS9SS9$ )NF
AES_GCM_V1g      @)minutes   )r   plaintext_footercolumn_keysencryption_algorithmcache_lifetimedata_key_length_bits)peEncryptionConfigurationr   )r   r&   s     r   create_encryption_configr,   K   s,    %%) -  r   c                  *    [         R                  " SS9$ )Ni,  )r(   )r*   DecryptionConfigurationr   r   r   create_decryption_configr/   W   s    %%S99r   c           
          [         R                  " U R                  5        VVs0 s H  u  pXR                  S5      _M     snnS9$ s  snnf )NzUTF-8)custom_kms_conf)r*   KmsConnectionConfigitemsdecode)keyskey_namekeys      r   create_kms_connection_configr8   [   sK    !! "&
!- jj))!-
 
s   A
c                     [        U 5      $ Nr   )kms_connection_configurations    r   kms_factoryr<   d   s    9::r   c              #      #    U (       a  S v   g [         R                  " XS9   S v   S S S 5        g ! , (       d  f       g = f7f)Nmatch)pytestraises)success
error_typer?   s      r   cond_raisesrD   h   s%     ]]:3 433s   !A1	A
?Ac           	        ^ ^^ U(       aH  [        S0 [        D[        [        0D6m[        S0 [        D[        U/0D6mUR                  S5      S   nO[        m[        mS n SUUU 4S jjnT R                  nTR                  5        VVVs/ s H#  u  pVU  H  nUR                  S5      S   PM     M%     nnnnU Vs/ s H*  nXx;  d  M
  Ub  UR                  U S35      (       a  M(  UPM,     n	n[        U5      S:  d   e[        U	5      S:  d   e[        [        0n
TR                  5        VVs0 s H  u  p[U[        :w  d  M  X[_M     nnnU" XS5        U" XS5        U" XS5        U" XSS5        U" XSS5        U" XSS5        [        T5      S:  a  TR                  5        H  u  pU Hz  nUR                  S5      S   nTR                  5        VVs0 s H  u  p[U[        U4;   d  M  X[_M     nnnX:g  nX/-   nU" UU	S5        U" UUU5        U" UUS5        U" UUS5        M|     M     U" TU	S5        U" TUS5        U" TUS5        g s  snnnf s  snf s  snnf s  snnf )	N.r   Tc                   > Tn[        [        T5      n[        5       n[        U5      n[        U 5      n[        R
                  " [        5      n	[        R                  " XU5      n
[        R                  " XU5      n[        R                  R                  5       nUR                  U
S9n[        R                  " 5       nUR!                  S5        [        R"                  " TSUUUS9  [        R                  R                  5       n[$        R&                  " [(        SS9   [        R                  " SXS9  S S S 5        [        R*                  " US9n[        R                  R                  US	9n[-        U[.        S
S9   [        R                  " SXS9n[-        U[.        S
S9   TR1                  U5      R3                  UR5                  U5      5      (       d   e S S S 5        S S S 5        U	R7                  X5      n[        R*                  " US9n[        R                  R                  US	9n[-        U[.        S
S9   [        R                  " SXS9n[-        U[.        S
S9   TR1                  U5      R3                  UR5                  U5      5      (       d   e S S S 5        S S S 5        g ! , (       d  f       GN{= f! , (       d  f       N= f! , (       d  f       N= f! , (       d  f       NK= f! , (       d  f       g = f)Nencryption_config/sample_datasetdatabase_dirformatfile_options
filesystemzno decryptionr>   rO   rQ   decryption_configdefault_fragment_scan_optionszUnknown master key)decryption_properties)r,   FOOTER_KEY_NAMEr/   r8   r*   CryptoFactoryr<   dsParquetEncryptionConfigParquetDecryptionConfigr   datasetParquetFileFormatmake_write_optionsfs_MockFileSystem
create_dirwrite_datasetr@   rA   IOErrorParquetFragmentScanOptionsrD   
ValueErrorselectequalsto_tablefile_decryption_properties)	read_keysread_columnsto_table_successdataset_success
write_keysrI   rT   encrypt_kms_connection_configdecrypt_kms_connection_configcrypto_factoryparquet_encryption_cfgparquet_decryption_cfgpformatwrite_optionsmockfspq_scan_optsr]   rW   r&   r5   r   s                     r   assert_decrypts>do_test_dataset_encryption_decryption.<locals>.assert_decrypts}   s    
4_kR46(DZ(P%(DY(O%))+6!#!;!;;L"
 "$!;!;;L"

 **..0224 3 
 ##%#
%&	
 **..0]]7*:;JJ'K < 444
 **..*6 / 
 *<PQjj!1'UG-zAUV||L1889I9I,9WXXXX W R !/ I I)!>44"7
 **..*6 / 
 *<PQjj!1'UG-zAUV||L1889I9I,9WXXXX W RQ3 <; WV RQ" WV RQsT   J(<%K!7J:K:%K-7KK-(
J7:
K	K
K
K*	&K--
K;F   r   )T)dictKEYSEXTRA_COL_KEY_NAMEEXTRA_COL_KEYCOLUMN_KEYSsplitcolumn_namesr3   
startswithlenrX   
FOOTER_KEY)r   extra_column_pathextra_column_namery   all_column_namesr6   r   column_nameencrypted_column_namesplaintext_column_namesfooter_key_onlyr7   column_keys_onlycolumn_key_namecolumn_key_column_namesencrypted_column_namerk   plaintext_and_one_successplaintext_and_oner&   r5   s   `                  @@r   %do_test_dataset_encryption_decryptionr   q   s   BdB1=ABV[V-?BSAT,UV-33C8;!  	DY DYN ))<G<M<M<O?<O"8(1=+ *//4Q71= 8<O  ? 2BU1A+!,!J *  18#4#?#?;-q@Q#R	 *1A  U
 %&***%&***&
3O-1ZZ\8-9MH#6 &-9  8 OTBOUCOu= $eUK$eUK$uE ;!8C8I8I8K4O)@% )>(C(CC(H(K% 37**,P2> (_o,N N +X]2>  P -B,V)$:=T$T!	+A4H	+<>WX	+A5I	+;UC% *A 9L, D0$7D0$7D*D1m?U84Ps0   *I>	II)I2III!
.I!
z+Parquet Encryption is not currently enabled)reasonc                  *    [        [        5       5        g r:   )r   r    r   r   r   "test_dataset_encryption_decryptionr      s     **=*?@r   r   listzlist.list.elementc                     [         R                  " / SQ/ SQ/ SQS/S/S//[         R                  " [         R                  " 5       5      S9n[	        5       R                  SU5      n[        X 5        g )	N)r{   r      )r   r      )      	   typer   )r   arraylist_int32r    append_columnr   )r   	list_datar   s      r   test_list_encryption_decryptionr     sY    
 	Iy2$rd;XXbhhj!I  !//	BE)%=r   )mapzmap.key_value.keyzmap.key_value.valuec                    [         R                  " [         R                  " 5       [         R                  " 5       5      n[         R                  " SS/SS/SS/S// / /US9n[        5       R                  S	U5      n[        X05        g )
N)k1r{   )k2r   )r   r   )k3r   )r   r   )r   r   )k4r   r   r   )r   map_stringr   r   r    r   r   )r   map_typemap_datar   s       r   test_map_encryption_decryptionr     sv     wwryy{BHHJ/Hxx	"Y	$:Y	<RKR	
 H  !//x@E)%=r   )structz	struct.f1z	struct.f2c                    S[         R                  " 5       4S[         R                  " 5       4/n[         R                  " U5      n[         R                  " / SQUS9n[        5       R                  SU5      n[        X@5        g )Nf1f2)r{   oner   twor   threer   fourr   fiver   sixr   r   )r   r   r   r   r   r    r   r   )r   struct_fieldsstruct_typestruct_datar   s        r   !test_struct_encryption_decryptionr   (  se    
 BHHJ'$		)<=M))M*K((TK  !//+FE)%=r   )colzcol.list.element.key_value.keyz#col.list.element.key_value.value.f1z#col.list.element.key_value.value.f2c                    S[         R                  " 5       4S[         R                  " 5       4/n[         R                  " U5      nSnSnSnSnSnSn[         R                  " [         R                  " 5       U5      n	X4S	.n
S
U0nSU0nXxS.n[         R
                  " U	5      nX/nU/nU/n[         R                  " UUS US S /US9/n[        5       R                  SU5      n[        UU 5        g )Nr   r   r   r   r   r   r   r   )r{   r   r   r   )r   r   r   r   )
r   r   r   r   r   r   r   r    r   r   )r   r   r   struct1struct2struct3struct4struct5struct6r   map1map2map3map4	list_typelist1list2list3r   r   s                       r   &test_deep_nested_encryption_decryptionr   8  s     BHHJ'$		)<=M))M*KGGGGGGwwrxxz;/H#Dw<Dw<D#D"ILEFEFE5%udDA	RSI!//yAE)%=r   z'Parquet Encryption is currently enabledc                      [         R                  R                  5       n [        R                  " [
        5         U R                  SS9nSSS5        g! , (       d  f       g= f)zTest write_dataset with ParquetFileFormat and test if an exception is thrown
if you try to set encryption_config using make_write_optionsz
some valuerH   N)r   r]   r^   r@   rA   NotImplementedErrorr_   )ru   _s     r   -test_write_dataset_parquet_without_encryptionr   _  sB     jj**,G	*	+&&&F 
,	+	+s   A
A c            	        ^  " S S[         R                  5      mSn [        R                  R	                  [        R
                  " [        U 5       Vs/ s H  n[        R                  " 5       PM     sn[        R                  " 5       S9/S/S9n[         R                  " 5       n[         R                  " U4S j5      n[         R                  " SSS/0S	S
SS9n[        R                  " XCU5      n[        R                  " XC[         R                  " 5       5      n[        R                   " US9n[        R"                  " US9n	U	R%                  US9n
UR'                  U5      n[(        R*                  " 5       nUR-                  S5        Sn[        R.                  " X-U	XS9  US-   n[0        R2                  " XUS9R5                  5       nX/:X  d   e[        R6                  " XUS9nUR9                  5       nX/:X  d   egs  snf )z9Test encryption and decryption of a large number of rows.c                   @    \ rS rSrS\S\S\4S jrS\S\S\4S jrSrg	)
;test_large_row_encryption_decryption.<locals>.NoOpKmsClientit  	key_bytesr   returnc                 2    [         R                  " U5      nU$ r:   )base64	b64encode)selfr   r   bs       r   wrap_keyDtest_large_row_encryption_decryption.<locals>.NoOpKmsClient.wrap_keyu  s      +AHr   wrapped_keyc                 2    [         R                  " U5      nU$ r:   )r   	b64decode)r   r   r   r   s       r   
unwrap_keyFtest_large_row_encryption_decryption.<locals>.NoOpKmsClient.unwrap_keyy  s      -AHr   r   N)	__name__
__module____qualname____firstlineno__bytesstrr   r   __static_attributes__r   r   r   NoOpKmsClientr   t  s6    	e 	 	 		% 	C 	E 	r   r   i  r   foo)namesc                    > T" 5       $ r:   r   )r   r   s    r   <lambda>6test_large_row_encryption_decryption.<locals>.<lambda>  s    r   UNIMPORTANT_KEYTF   )r   r&   double_wrappingr%   r)   rS   rU   rH   rJ   zlarge-row-test-dataset)rO   rP   rQ   z/part-0.parquetrW   rQ   rR   N)r*   	KmsClientr   Tablefrom_arraysr   rangerandomfloat32r2   rY   r+   rZ   r[   r\   r.   re   r^   r_   rj   r`   ra   rb   rc   pqParquetFilereadr]   ri   )	row_countr   r   
kms_configrr   rI   
pqe_config
pqd_configscan_optionsfile_formatrv   rj   rw   path	file_path	new_tabler]   r   s                    @r   $test_large_row_encryption_decryptionr  n  s     IHH  	&+I&67&6V]]_&67

 	 '	 ! E '')J%%&?@N22$&0  ++$5J ++B$>$>$@J 00:NL&&\RK22Z2PM!/!J!J:!V!F
c#DU"/D ((I df  jjfEG  "IS 8s   G=c                     [        5       n [        5       n[        5       n[        5       n[        R
                  " [        5      n[        R                  " XCU5      n[        R                  " XCU5      n[        R                  R                  5       nUR                  USS/S9n[        R                  " 5       n	U	R!                  S5        [        R"                  " U SUUU	S9  [        R$                  " US9n
[        R                  R                  U
S9n[        R                  " SXyS	9nUR'                  5        GH  nUR)                  X2UR*                  U	5      n[,        R.                  " UR*                  UU	S
9 n[1        UR2                  R4                  5       GH%  nUR2                  R7                  U5      nUR9                  S5      R:                  c   eUR9                  S5      R:                  R<                  S:X  d   eUR9                  S5      R:                  R>                  S:X  d   eUR9                  S5      R:                  c   eUR9                  S5      R:                  R<                  S:X  d   eUR9                  S5      R:                  R>                  S:X  d   eUR9                  S5      R:                  c  GM&   e   S S S 5        GM     g ! , (       d  f       GM  = f)Nr   r   )rI   write_statisticsrJ   rK   rL   rS   rU   rR   r   r   r   r   r{   r   r   ) r    r,   r/   r8   r*   rY   r<   rZ   r[   r\   r   r]   r^   r_   r`   ra   rb   rc   re   get_fragmentsrj   r  r  r  r  metadatanum_row_groups	row_groupcolumn
statisticsminmax)r   rI   rT   kms_connection_configrr   rs   rt   ru   rv   rw   rx   r]   fragmentrW   parquet_filerg_idxr  s                    r   7test_dataset_encryption_with_selected_column_statisticsr     s     !E02028:%%k2N77/@  77/@ jj**,G..0 (+ / M
 !F
c!" 000L jj***VGjj)'MG))+ . I I!hmmV!M^^MM"7
  5 5 D DE(11;;FC	 ''*55AAA ''*5599TAAA ''*5599TAAA ''*55AAA ''*5599Q>>> ''*5599S@@@ ''*55=== F
 
 ,
 
s   6EK<K
K	r:   )4r   
contextlibr   datetimer   r  
pyarrow.fsr`   pyarrowr   r@   encryption_unavailablepyarrow.parquetparquetr  pyarrow.datasetr]   rZ   ImportError pyarrow.tests.parquet.encryptionr   pyarrow.parquet.encryption
encryptionr*   mark
pytestmarkr   rX   COL_KEYCOL_KEY_NAMEr}   r   r~   COLUMNSr   r    r,   r/   r8   r<   rD   r   skipifr   parametrizer   r   r   r   r   r  r   r   r   r   <module>r4     s  $  %        
"B++ [[  
 !

\7;# 
&h12" )8[ 	: '+ ;  J2Z #P  AA #P   1D(EF> G> 8   F>	> #P   (LM	> N	> 8   >	>4 'P  	G	G #P  88v #P  =>=>_  	B	B  "!"s"   H! H2 !
H/.H/2H>=H>