
    <>iT                    *   S SK r S SKrS SKrS SKrS SKrS SKrS SKrS SKrS SKrS SK	r	S SK
r
S SKJr  S SKJr   S SKrS SKrS SKrS SKJr  S SKrS SKrS SKJr  S SKrS SKJr  S SKJ r J!r!J"r"J#r#J$r$   S SK%r& S SK'J(r)   S SK*J+r,  \RZ                  RP                  r. " S S5      r/S r0S	 r1S
 r2\Rf                  S 5       r4\Rf                  S 5       r5\Rf                  S 5       r6\Rf                  " SS9S 5       r7\Rf                  S 5       r(\Rf                  " SS/SS/S9S 5       r8\RZ                  RV                  S 5       r9S r:\RZ                  RV                  S 5       r;S r<S r=\RZ                  RV                  S 5       r>\RZ                  RV                  S 5       r?\RZ                  RV                  S 5       r@\RZ                  RV                  S  5       rA\RZ                  RV                  S! 5       rB\RZ                  RV                  S" 5       rCS# rDS$ rES% rF\RZ                  R                  S&/ S'Q5      S(\HS)\IS*\I4S+ j5       rJS, rKS- rL\RZ                  RV                  S. 5       rM\RZ                  RV                  S/ 5       rN\RZ                  RV                  S0 5       rOS1 rPS2 rQ\RZ                  R                  S3\R                  " S4SS59S6S7//5      \RZ                  R                  S8SS/5      \RZ                  RV                  S9 5       5       5       rS\RZ                  RV                  S: 5       rT\RZ                  RV                  \RZ                  R                  S; 5       5       rVS< rWS= rX\RZ                  RV                  S> 5       rY\RZ                  RV                  GSS? j5       rZ\RZ                  RV                  S@ 5       r[\RZ                  RJ                  \RZ                  RV                  SA 5       5       r\\RZ                  RV                  SB 5       r]\RZ                  RV                  SC 5       r^\RZ                  RV                  SD 5       r_\RZ                  RJ                  \RZ                  RV                  SE 5       5       r`\RZ                  RV                  SF 5       ra\RZ                  RV                  SG 5       rbGSSH jrc\RZ                  RJ                  \RZ                  RV                  SI 5       5       rd\RZ                  RV                  SJ 5       re\RZ                  RJ                  \RZ                  RV                  SK 5       5       rf\RZ                  RV                  SL 5       rg\RZ                  RV                  SM 5       rh\RZ                  RV                  SN 5       ri\RZ                  RV                  SO 5       rj\RZ                  RV                  SP 5       rk\RZ                  RV                  SQ 5       rl\RZ                  RJ                  \RZ                  RV                  SR 5       5       rm\RZ                  RV                  \RZ                  R                  SSST SU /5      SV 5       5       rn\RZ                  RV                  \RZ                  R                  SWSS/5      \RZ                  R                  SSSX SY /5      SZ 5       5       5       ro\RZ                  R                  SSS[ S\ /5      S] 5       rp\RZ                  R                  SSS^ S_ /5      S` 5       rqSa rrSb rs\RZ                  RV                  \RZ                  RJ                  Sc 5       5       rtSd ruSe rvSf rwSg rxSh rySi rzSj r{\RZ                  RV                  Sk 5       r|Sl r}GSSm jr~Sn rSo rSp r\RZ                  RV                  Sq 5       r\RZ                  RV                  Sr 5       r\RZ                  RV                  Ss 5       r\RZ                  RV                  St 5       r\RZ                  RV                  Su 5       r\RZ                  RV                  Sv 5       r\RZ                  RV                  Sw 5       r\RZ                  RV                  Sx 5       r\RZ                  RV                  Sy 5       r\RZ                  RV                  Sz 5       rS{ rS| rS} rS~ r\RZ                  R                  SSS/5      S 5       rS r\RZ                  RV                  S 5       r\RZ                  RV                  S 5       r\RZ                  RV                  S 5       r\RZ                  RV                  S 5       rS rS r\RZ                  RV                  \RZ                  R                  SSS/5      \RZ                  R                  SSS/5      \RZ                  R                  SWSS/5      \RZ                  R                  S/ SQ/ SQ4/ SQ/ SQ4/ SQ/ SQ4/ SQ/ SQ4/ SQ/ SQ4/ SQ/ SQ4/ SQ/ SQ4/5      S 5       5       5       5       5       r\RZ                  RJ                  S 5       r\Rf                  S 5       r\RZ                  RV                  \RZ                  R                  S 5       5       r\RZ                  RV                  \RZ                  R                  S 5       5       r\RZ                  RV                  \RZ                  R                  S 5       5       r\RZ                  RV                  \RZ                  R                  S 5       5       r\RZ                  RV                  S 5       r\RZ                  RV                  S 5       r\RZ                  RJ                  S 5       r\RZ                  RV                  S 5       r\RZ                  RV                  S 5       r\RZ                  RV                  S 5       rS rS rS rS r\RZ                  RV                  S 5       r\RZ                  RV                  S 5       rS r\RZ                  GRX                  S 5       r\RZ                  GRX                  S 5       rS r\RZ                  GRX                  S 5       r\RZ                  RJ                  S 5       r\RZ                  RJ                  \RZ                  R                  S/ SQ5      S 5       5       rS rS rS r\RZ                  RJ                  S 5       r\RZ                  RJ                  S 5       r\RZ                  RJ                  S 5       rS rS rS r\RZ                  RJ                  \RZ                  R                  S/ SQ5      S 5       5       rS r\RZ                  RV                  \RZ                  RJ                  S 5       5       r\RZ                  RV                  \RZ                  RJ                  \RZ                  GR                  \GR                  S:H  SS9S 5       5       5       r\RZ                  RV                  \RZ                  RJ                  S 5       5       r\RZ                  RV                  S 5       r\RZ                  RV                  \RZ                  RJ                  S 5       5       rS rS r\RZ                  RV                  \RZ                  RJ                  S 5       5       r\RZ                  RV                  \RZ                  RJ                  S 5       5       r\RZ                  RV                  \RZ                  RJ                  S 5       5       r\RZ                  RV                  \RZ                  RJ                  S 5       5       r\RZ                  RV                  S 5       r\RZ                  RV                  \RZ                  RJ                  S 5       5       r\RZ                  RV                  \RZ                  RJ                  S 5       5       rS r\RZ                  RJ                  \RZ                  RV                  S 5       5       r\RZ                  RV                  \RZ                  RJ                  S 5       5       rS r GSS jr\RZ                  RV                  S 5       r\RZ                  RV                  \RZ                  RJ                  S 5       5       rS rS rS r\RZ                  RV                  \RZ                  R                  S 5       5       rS r\RZ                  RJ                  S 5       rGSS jrS rS r\RZ                  RV                  S 5       r\RZ                  RV                  S 5       r\RZ                  RV                  S 5       r\RZ                  RV                  S 5       r\RZ                  RV                  \RZ                  RJ                  S 5       5       r\RZ                  RV                  \RZ                  RJ                  S 5       5       r\RZ                  RV                  \RZ                  RJ                  S 5       5       rS rS rS rS rS r\RZ                  R                  \RZ                  RV                  S 5       5       rS r\RZ                  RV                  S 5       r\RZ                  RV                  S 5       r\RZ                  RV                  \RZ                  RJ                  S 5       5       rS r\RZ                  RV                  S 5       r\RZ                  RV                  \RZ                  R                  S 5       5       rSr\RZ                  RV                  \RZ                  R                  S 5       5       r\RZ                  RV                  S 5       r\RZ                  RP                  S 5       r\RZ                  RP                  S 5       r\RZ                  RP                  S 5       r\RZ                  RP                  S 5       r\RZ                  RP                  S 5       r\RZ                  RP                  S 5       r\RZ                  RP                  S 5       r\RZ                  R                  SSS/5      S 5       r\RZ                  R                  SSS/5      GS  5       rGS rGS r\RZ                  R                  GSGS5      GS 5       Gr GS Gr\RZ                  RV                  GS 5       GrGS GrGS	 Gr\RZ                  R                  SSS/5      GS
 5       GrGS GrGS GrGS Grg! \ a    Sr GNf = f! \ a    Sr& GNf = f! \ a    Sr) GNf = f! \ a    Sr, GNf = f(      N)copytree)quote)is_threading_enabled)FSProtocolClassProxyHandler_configure_s3_limited_user_filesystem_uri
change_cwdc                   $    \ rS rSrS rSS jrSrg)TableStreamWrapperE   c                     Xl         g Ntable)selfr   s     Y/var/www/html/land-tabula/venv/lib/python3.13/site-packages/pyarrow/tests/test_dataset.py__init__TableStreamWrapper.__init__F   s    
    Nc                 8    U R                   R                  U5      $ r   )r   __arrow_c_stream__)r   requested_schemas     r   r   %TableStreamWrapper.__arrow_c_stream__I   s    zz,,-=>>r   r   r   )__name__
__module____qualname____firstlineno__r   r   __static_attributes__ r   r   r   r   E   s    ?r   r   c           	      *   SS K nSS KnUR                   " SSS5      nUR                  " SS9nUR                  / SQ5      n/ n[	        U 5       H-  nUR                  X7[        U5      [        U5      45        X4-  nM/     [        R                  " U/ SQS9$ )	Nr   i        )days)greenblueyellowredorange)dateindexvaluecolorcolumns)
datetime	itertools	timedeltacyclerangeappendfloatnextpd	DataFrame)nr0   r1   dayintervalcolorsdatais           r   _generate_datar@   M   s    


D!Q
'C!!q)H__IJFD1XSU1XtF|45  <<&IJJr   c           
         [         R                  " [         R                  " S[         R                  " 5       5      [         R                  " S[         R                  " 5       5      [         R                  " S[         R
                  " 5       5      [         R                  " S[         R                  " 5       5      /5      n[         R                  R                  XSS9nUR                  5       $ )Nr*   r+   r,   r-   F)schemapreserve_index)
parB   fielddate32int64float64stringTablefrom_pandasreplace_schema_metadata)dfrB   r   s      r   _table_from_pandasrN   ]   s    YY
%
"((*%
"**,'
"))+&	 F HH  5 IE((**r   c                 |   U R                  5        H  nUR                  5        n[        U[        R                  5      (       d   eUR
                  (       a   eUR                  5       (       d   eUR                  5       (       d   eUR                  5       (       a   e S S S 5        M     g ! , (       d  f       M  = fr   )	get_fragmentsopen
isinstancerD   
NativeFileclosedseekablereadablewritable)datasetfragmentnfs      r   +assert_dataset_fragment_convenience_methodsr[   h   s    ))+]]_b"--0000yy =;;== =;;== ={{}}$$} _ ,_s   A:B,,
B;	c                     [         R                  " 5       n SS/n[        U5       GH  u  p#U SU S3nU R                  U5        U R	                  U5       n[        [        S5      5      [        [        [        [        S5      5      5      [        [        [        [        S5      5      5      U/S-  [        S5       Vs/ s H  ofS-  [        US-  5      S.PM     sn/n[        R                  " S[        R                  " 5       4S	[        R                  " 5       4S
[        R                  " 5       4S[        R                  " 5       4S[        R                  " [        R                  " 5       [        R                  " 5       S.5      4/5      n[        R                   " XxS9n	[        R"                  R%                  U	/5      n
[&        R(                  " X5        S S S 5        GM     U $ s  snf ! , (       d  f       GM  = f)Nzsubdir/1/xxxzsubdir/2/yyyz/file.parquetr#      abi64f64strconststructrB   )fs_MockFileSystem	enumerate
create_diropen_output_streamlistr4   mapr6   rd   rD   rB   rG   rH   rI   rf   record_batchrJ   from_batchespqwrite_table)mockfsdirectoriesr?   	directorypathoutjr>   rB   batchr   s              r   rs   rs   s   s   !F 	K
 "+.E!H-)$&&t,U1XSa)*SeAh'(a8=aA11u3q1u:.AD YY
#

%		$"((*%299288:BIIK%HIJ  F OOD8EHH))5'2ENN5&% -, /. M B -,s    A(G*=G%C<G*%G**
G:	c            
         [         R                  " 5       n [        R                  " [        R                  " SS/[        R
                  " 5       S9[        R                  R                  [        R                  " SS/[        R
                  " 5       S9SS/5      S.5      n[        R                  " [        R                  " SS	/[        R                  " 5       S9[        R                  R                  [        R                  " SS/[        R                  " 5       S9S
S/5      S.5      nU R                  S5        SnU R                  U5       n[        R                  " X5        SSS5        SnU R                  U5       n[        R                  " X$5        SSS5        XU4$ ! , (       d  f       ND= f! , (       d  f       N#= f)z
Creates a _MockFileSystem with two parquet files that have promotable schemas.
- file1: value: int8, dictionary: dictionary<int8, string>
- file2: value: uint16, dictionary: dictionary<int16, string>
r"      typer   r`   ra   r,   
dictionaryr^      dcz
subdir/zzzzsubdir/zzz/file1.parquetNzsubdir/zzz/file2.parquet)rh   ri   rD   r   arrayint8DictionaryArrayfrom_arraysuint16int16rk   rl   rq   rr   )rs   table1table2path1rw   path2s         r   promotable_mockfsr      sM    !FXX1a&rwwy1((44HHaV"''),#J
 F XX1a&ryy{3((44HHaV"((*-#J
 F l#&E		"	"5	)S
v# 
* 'E		"	"5	)S
v# 
* % 
*	) 
*	)s   F!>F2!
F/2
G c                    ^^^	 SSK JnJn  SSKJn  U" 5       mU4S jm[        5       m	UU	4S jnU R                  USU5        U" U" T5      5      n[        R                  UU	4S j5       nXV4$ )	Nr   )LocalFileSystemPyFileSystemr"   )r   c                 d   > U  Vs1 s H  nTR                  [        U5      5      iM     sn$ s  snf r   )normalize_pathrd   )pathsplocalfss     r   
normalized#open_logging_fs.<locals>.normalized   s*    8=>1&&s1v.>>>s   $-c                    > TR                  [        U5      5      nTR                  U5        U R                  R	                  U5      $ r   )r   rd   add_fsopen_input_file)r   rv   r   openeds     r   r   (open_logging_fs.<locals>.open_input_file   s8    %%c$i0

4xx''--r   r   c              3      >#    TR                  5          S v   T" T5      T" U 5      :X  d   eg ! T" T5      T" U 5      :X  d   ef = f7fr   )clear)expected_openedr   r   s    r   assert_opens%open_logging_fs.<locals>.assert_opens   sH     	Ef%O)DDDD:f%O)DDDDs   A. AAA)	
pyarrow.fsr   r   test_fsr   setsetattr
contextlibcontextmanager)
monkeypatchr   r   r   r   rh   r   r   r   r   s
          @@@r   open_logging_fsr      sm    8%G? UF. &7I	l7+	,BE E r   module)scopec           
      4   U R                   R                  R                  S5        U R                   R                  R                  S5        [        S5      n[        R
                  " 5       n[        U5      n[        SX3S-  5       Vs/ s H  oAR                  XDUS-  -    PM     snu  pVpxUR                  S5        [        U5      n[        [        SX3S-  5       Vs/ s H  oER                  XDUS-  -    PM     sn5       HE  u  pISU S	3n
UR                  U
5       n[        R                  " [        U	5      U5        S S S 5        MG     UR                  S
5        UR                  UR                   R"                  R$                  UR&                  /5       Hc  u  pSUS    SUS    3nU S3n
UR                  U5        UR                  U
5       n[        R                  " [        U	5      U5        S S S 5        Me     UR                  S5        UR                  UR                   R"                  R(                  UR                   R"                  R*                  /5       Hc  u  pSUS    SUS    3nU S3n
UR                  U5        UR                  U
5       n[        R                  " [        U	5      U5        S S S 5        Me     UR                  S5        UR                  S5       HZ  u  pSU 3nU S3n
UR                  U5        UR                  U
5       n[        R                  " [        U	5      U5        S S S 5        M\     U$ s  snf s  snf ! , (       d  f       GM_  = f! , (       d  f       GM  = f! , (       d  f       GM)  = f! , (       d  f       M  = f)Npandasparquet  r   r   plain
   zplain/chunk-r]   rB   zschema//r"   z/chunk.parquethivez
hive/year=z/month=
hive_colorr-   zhive_color/color=)configpyarrowrequiresr@   rh   ri   lenr4   ilocrk   rj   rl   rq   rr   rN   groupbyr*   dt	dayofweekr-   yearmonth)requestrM   rs   r:   r?   df_adf_bdf_cdf_dchunkrv   rw   partfolders                 r   multisourcefsr      s   NN##H-NN##I.		B!F 	BA9>q!T9JK9JAgga!Q$/9JKD gD	AU1aB=OP=OyyQU73=OPQaS)&&t,NN-e4c: -, R h||TYY\\%;%;TZZ$HI47)1T!WI.(&!&&t,NN-e4c: -,	 J f||TYY\\%6%6		8J8J$KLd1gYgd1gY7(&!&&t,NN-e4c: -,	 M l#||G,$TF+(&!&&t,NN-e4c: -,	 - MM L Q,, -, -, -,s<   MM
!M!M"!M5!N
M	"
M2	5
N	
N	c           
         [         R                  " 5       n[        R                  " SSS9n[         R                  " S5      n[         R
                  " [        R                  " [        R                  " S[        R                  " 5       5      [        R                  " S[        R                  " 5       5      /5      5      Ul        [         R                  " XX5      nUR                  5       $ )NsubdirT	recursivegroupkey)dsParquetFileFormatrh   FileSelectorFileSystemFactoryOptionsDirectoryPartitioningrD   rB   rE   int32rI   partitioningFileSystemDatasetFactoryfinish)rs   formatselectoroptionsfactorys        r   rX   rX     s    !!#Fx48H))(3G33
		HHWbhhj)HHUBIIK(
 	G ))&FLG>>r   TFthreadedserial)paramsidsc                 D   ^ U R                   m " U4S jS5      nU" 5       $ )zQ
Fixture which allows dataset scanning operations to be
run with/without threads
c                   R   > \ rS rSrU 4S jrU 4S jrS rS rS rS r	S r
S	 rS
rg)dataset_reader.<locals>.readeri)  c                    > TU l         g r   use_threads)r   r   s    r   r   'dataset_reader.<locals>.reader.__init__+  s    *Dr   c                 2   > SU;   a  [        S5      eTUS'   g )Nr   z9Invalid use of dataset_reader, do not specify use_threads)	Exception)r   kwargsr   s     r   _patch_kwargs,dataset_reader.<locals>.reader._patch_kwargs.  s)    &$& & %0F=!r   c                 H    U R                  U5        UR                  " S0 UD6$ Nr    )r   to_tabler   rX   r   s      r   r   'dataset_reader.<locals>.reader.to_table5  s$    v&##-f--r   c                 H    U R                  U5        UR                  " S0 UD6$ r   )r   
to_batchesr   s      r   r   )dataset_reader.<locals>.reader.to_batches9  $    v&%%///r   c                 H    U R                  U5        UR                  " S0 UD6$ r   )r   scannerr   s      r   r   &dataset_reader.<locals>.reader.scanner=  s"    v&??,V,,r   c                 J    U R                  U5        UR                  " U40 UD6$ r   )r   head)r   rX   num_rowsr   s       r   r   #dataset_reader.<locals>.reader.headA  s$    v&<<3F33r   c                 J    U R                  U5        UR                  " U40 UD6$ r   )r   take)r   rX   indicesr   s       r   r   #dataset_reader.<locals>.reader.takeE  s$    v&<<2622r   c                 H    U R                  U5        UR                  " S0 UD6$ r   )r   
count_rowsr   s      r   r  )dataset_reader.<locals>.reader.count_rowsI  r   r   r   N)r   r   r   r   r   r   r   r   r   r   r   r  r   r   s   r   readerr   )  s+    	+	0	.	0	-	4	3	0r   r  )param)r   r  r   s     @r   dataset_readerr    s"     --K"0 "0H 8Or   c           	      
   [         R                  " [         R                  " S[         R                  " 5       5      /5      n[        R
                  " 5       nSS/n[        SS5       Vs/ s H  n[        R                  " S5      U:H  PM     nn[        X55       VVs/ s H  u  pgUR                  X`U5      PM     nnn[        R                  " S5      [        R                  " S5      :H  n	[        R                  " XUX	S	9n
[        R                  R                  X1X XYS
9nX4 GHN  n[        U[        R                  5      (       d   e[        UR                  [        R
                  5      (       d   eUR                  R                  U	5      (       d   e[!        UR"                  5      [!        U5      :X  d   e[%        UR'                  5       5      n[        XU5       GHN  u  pnUR                  R                  U5      (       d   eUR(                  U:X  d   e[        UR                  [        R
                  5      (       d   e[        U[        R*                  5      (       d   eUR,                  S/:X  d   eUR.                  S:X  d   e[%        UR1                  5       5      nUR.                  [3        U5      s=:X  a  S:X  d   e   e[        US   [        R*                  5      (       d   eUS   R(                  U:X  d   eUS   R,                  S/:X  d   eUS   R.                  S:X  a  GMO   e   [%        UR'                  [        R                  " S5      S:H  S95      n[3        U5      S:X  a  GMO   e   [        R                  " XX S9nUR                  R                  [        R                  " S5      5      (       d   e[        R                  R                  X1X S9nUR                  R                  [        R                  " S5      5      (       d   eUR'                  5        H9  nUR                  R                  [        R                  " S5      5      (       a  M9   e   [4        R6                  " [8        SS9   [        R                  " XU5        S S S 5        [4        R6                  " [8        SS9   [        R                  " XUSS9  S S S 5        [4        R6                  " [8        SS9   [        R                  R                  XS9  S S S 5        g s  snf s  snnf ! , (       d  f       N= f! , (       d  f       Nj= f! , (       d  f       g = f)Nre   subdir/1/xxx/file0.parquetsubdir/2/yyy/file1.parquetr"   r^   r   leveli9  )rB   r   
filesystemroot_partition)rB   r   r
  
partitionsr  r   filterr{   rB   r   r
  Tzincorrect typematch)rB   r   r  r   )rD   rB   rE   rG   r   r   r4   zipmake_fragmentscalarFileSystemDataset
from_pathsrR   r   partition_expressionequalsr   filesrm   rP   rv   ParquetFileFragment
row_groupsnum_row_groupssplit_by_row_groupr   pytestraises	TypeError)rs   rB   file_formatr   xr  rv   r   	fragmentsr  dataset_from_fragmentsdataset_from_pathsrX   rY   	partitionrow_group_fragmentss                   r   test_filesystem_datasetr)  P  sx   YY
"((*% F &&(K)+GHE16q!=A"((6"a'J=#&u#9;#9ZT **4>#9  ;XXg&"))D/9N11 --88[ 9 
 +?'2#7#78888'.."*>*>????++22>BBBB7==!SZ///..01	),YE)J%H0077	BBBB==D(((hoor/C/CDDDDh(>(>????&&1#---**a///"&x'B'B'D"E**c2E.FK!KKKKK1!4b6L6LMMMM&q)..$666&q)44;;;&q)88A=== *K ..bhhw6G16L.MN	9~"""/ @4 ""G ''..ryy???? ""--[ . G ''..ryy????))+,,33BIIdODDDD , 
y(8	9
YV< 
: 
y(8	9
Y$/	C 
: 
y(8	9
''	'F 
:	9{ >;j 
:	9 
:	9 
:	9s0   '#TT8T1T,)T=
T),
T:=
Uc                    [         R                  " [         R                  " S[         R                  " 5       5      /5      n[        R
                  " 5       nS/n[        R                  R                  X1U[        R                  " 5       S9nUR                  5         [        R                  " [        5         U R                  U5        S S S 5        g ! , (       d  f       g = f)Nf1znonexistingfile.arrowr  )rD   rB   rE   rG   r   IpcFileFormatr  r  rh   r   rP   r  r   FileNotFoundErrorr   )r  rB   r"  r   rX   s        r   1test_filesystem_dataset_no_filesystem_interactionr.    s    YY
rxxz" F ""$K$%E ""--[%%' . G  
(	)( 
*	)	)s   4C
Cc                    [        U [        R                  5      (       d   e[        U R                  [        R
                  5      (       d   e[        R                  " S5      n[        R                  " [        SS9   U R                  US9  S S S 5        [        R                  " S5      S-   n[        R                  " [        SS9   U R                  US9  S S S 5        [        R                  " / SQ[        R                  " 5       S9n[        R                  " / SQ[        R                  " 5       S9nUR                  U 5       Hr  n[        U[        R                  5      (       d   eUR!                  S5      R#                  U5      (       d   eUR!                  S5      R#                  U5      (       a  Mr   e   UR%                  U 5      R'                  5        HO  n[        U[        R(                  5      (       d   e[        UR*                  [        R,                  5      (       a  MO   e   UR                  U 5      n[        U[        R.                  5      (       d   e[1        U5      S	:X  d   e[        R                  " S5      S:H  nU R                  S
US9n	U	R3                  S5      R5                  5       n	U	S   SS/:X  d   eU	S   SS/:X  d   e[7        U	S   5      SS/:X  d   e[7        U	S   5      SS/:X  d   e[        R                  " S5      S:H  nU R                  S
US9n	U	R3                  S5      R5                  5       n	U	S   / SQ:X  d   eU	S   / SQ:X  d   eU	S   / SQ:X  d   eU	S   / SQ:X  d   e[        R                  " S5      [        R                  " S5      [        R                  " S5      S:H  S.n
U R                  S
U
S9n	U	R3                  S5      R5                  5       n	[9        U	5      / SQ:X  d   eU	S   / SQ:X  d   eU	S   / SQ:X  d   eU	S   / SQ:X  d   e[;        U 5        g ! , (       d  f       GN= f! , (       d  f       GNu= f)Nrb   zmust evaluate to boolr  r  r"   r   r"   r{   r^   r   r|   r   r   T)r   r  r   rc         ?r{   r   xxxyyy)rf   ra   1)r"   r   r"   r   )r1        @r1  r5  )r"   r"   r{   r{   )r2  r2  r3  r3  )rb   rc   new)r   r/   )
r   r   r"   r"   r{   r{   r^   r^   r   r   )
        r7  r1  r1         @r8        @r9  r5  r5  r6  )
FFTTFFFFTT)rR   r   DatasetrB   rD   SchemarE   r  r   r!  r   r   rG   rH   r   RecordBatchcolumnr  r   scan_batchesTaggedRecordBatchrY   FragmentrJ   r   sort_by	to_pydictsortedrm   r[   )rX   r  non_boolean_exprnon_boolean_expr2expected_i64expected_f64ry   r   	conditionresult
projections              r   test_datasetrK    s   grzz****gnnbii0000xx	y(?	@ 01 
A !+	y(?	@ 12 
A 88O"((*=L88O"**,?L**73%0000||A%%l3333||A%%l3333 4
  ''0==?%!5!56666%.."++6666 @ ##G,EeRXX&&&&u:1$I$yAF^^G$..0F%=QF"""%=RH$$$&/"q!f,,,&- UEN222 )S0I$yAF^^G$..0F%=L(((%=0000'?l***%=8888 xxxxxx(C/J
 $
CF^^E",,.F<0000%=::::%= ; ; ; ;%= 7 7 7 7/8u 
A	@ 
A	@s   <P2Q2
Q
Qc                 ^   U u  pn[         R                  " XU/[         R                  " 5       5      n[        R                  " [
        R                  SS9   UR                  5         S S S 5        UR                  SS9n[
        R                  " [
        R                  " S[
        R                  " 5       5      [
        R                  " S[
        R                  " [
        R                  " 5       [
        R                  " 5       5      5      /5      nUR                  U5      (       d   eUR                  U5      nUR!                  5       nUR#                  SS9  [
        R$                  " [
        R&                  " S	S
/SS//[
        R                  " 5       S9[
        R&                  " [
        R(                  R+                  [
        R,                  " SS	/[
        R                  " 5       S9SS/5      [
        R(                  R+                  [
        R,                  " S	S/[
        R                  " 5       S9SS/5      /5      S.5      n	UR                  U	5      (       d   eUR                  SS	S9n
U
R                  S5      R.                  [
        R                  " 5       :w  d   eg ! , (       d  f       GN>= f)NzCUnable to merge: Field value has incompatible types: int8 vs uint16r  
permissivepromote_optionsr,   r   Tfullr"   r{   r^   r   r|   r   r`   ra   r   r   r~   )rO  r$  )r   r   r   r  r   rD   ArrowTypeErrorinspectrB   rE   r   r   r   rI   r  r   r   validater   chunked_arrayr   r   r   r}   )r   rs   r   r   r   rB   expected_schemarX   r   expected_tableinspected_schema_one_frags              r   -test_dataset_factory_inspect_schema_promotionrY    s   ,F5)) 4 4 6G 

S
 		
 __\_:Fii
"((*%
r}}RXXZEF	! O ==))))nnV$GE	NNNXX!!Aq6Aq6"2D&&**!Qbhhj1c
 **!Qbhhj1c
	(
 	 N <<'''' !($ !0 !3$**7388BHHJFFFG
 
s   J
J,c                    U u  pn[         R                  " XU/[         R                  " 5       5      n[        R                  " [
        SS9   UR                  SS9  S S S 5        [        R                  " [
        SS9   UR                  SS9  S S S 5        [        R                  " [
        SS9   UR                  S	S9  S S S 5        g ! , (       d  f       Nq= f! , (       d  f       NQ= f! , (       d  f       g = f)
Nz#Invalid promote_options: bad_optionr  
bad_optionrN  z<Fragment count must be a non-negative int or None; got 'one'one)r$  z9Fragment count must be a non-negative int or None; got -1)r   r   r   r  r   
ValueErrorrS  )r   rs   r   r   r   s        r   'test_dataset_factory_inspect_bad_paramsr_    s    ,F5)) 4 4 6G 
z)N	O5 
P 
X
 	%(

 
U
 	"%
 
 
P	O
 


 
s$   C<C-C(
C
C%(
C6c                 ^    U R                  SSS9n[        U5      nUR                  S:X  d   eg )N      )fragment_readaheadbatch_readahead   )r   r7   num_columns)rX   r   ry   s      r   test_scanner_optionsrg  1  s4      B JGME!!!r   c                    UR                  U [        R                  " 5       S9n[        U[        R
                  5      (       d   e[        R                  " [        R                  5         UR                  U S/S9  S S S 5        UR                  U S/[        R                  " 5       S9nUR                  U R                  :X  d   eUR                  [        R                  " S[        R                  " 5       4/5      :X  d   e[        U[        R
                  5      (       d   eUR                  5       nUR                  5        H1  nUR                  UR                  :X  d   eUR                  S:X  a  M1   e   X2R!                  5       R#                  5       :X  d   eUR                  UR                  :X  d   e[%        UR&                  5       H@  n[        R(                  " U/5      nUR+                  U5      UR+                  U5      :X  a  M@   e   [        R                  " [        R,                  5         UR+                  [        R(                  " UR&                  /5      5        S S S 5        UR&                  UR/                  5       :X  d   eUR                  U / SQ[        R                  " 5       S9nUR                  5       n/ SQnUR0                  U:X  d   eUR3                  S5      nUS	   R5                  5       S
/S-  S/S-  -   :X  d   eUS   R5                  5       S/S-  S/S-  -   :X  d   eUS   R5                  5       S/S-  :X  d   eUS   R5                  5       S/S-  :X  d   eg ! , (       d  f       GN= f! , (       d  f       GN!= f)N)memory_poolunknownr.   rb   )r/   ri  r"   )
__filename__fragment_index__batch_index__last_in_fragmentrl  rk  r  r#   r  r   rm  r   rn  T)r   rD   default_memory_poolrR   r   Scannerr  r   ArrowInvaliddataset_schemarB   projected_schemarG   r   r   rf  	to_readerread_allr4   r   r   r   ArrowIndexErrorr  column_namesrA  	to_pylist)	rX   r  r   r   ry   r?   r   expected_namessorted_tables	            r   test_scannerr{  8  s,   $$R335 % 7Ggrzz****	r	'w< 
( $$Wug131G1G1I % KG!!W^^333##ryy5"((*2E1F'GGGGgrzz****E##%||w77777  A%%% & %%'002222<<7333335>>"((A3-zz'"gll7&;;;; # 
r))	*RXXu~~./0 
+ >>W//1111$$W 7M 241G1G1I	 % KG
 E=N///==!34L%//1	%&*	%&*	+, , , *+557QC!GqcAg<MNNN(224b@@@,-779dVb[HHHQ 
(	'& 
+	*s   )M+21M=+
M:=
Nc                    [         R                  " 5       n[         R                  " 5       n[         R                  " U5         UR	                  5       n[
        R                  R                  U 5      nUR                  5       nUR	                  5       U:  d   e [         R                  " U5        g ! [         R                  " U5        f = fr   )	rD   ro  system_memory_poolset_memory_poolbytes_allocatedr   rp  from_datasetr   )rX   old_poolpoolallocated_beforer   _s         r   test_scanner_memory_poolr  i  s     %%'H   "Dt%//1**))'2##%(8888
8$8$s   AB/ /Cc                    UR                  U S5      nU[        R                  R                  / U R                  S9:X  d   eUR                  U SS/S9R                  5       nUSS/0:X  d   eUR                  U SS/[        R                  " S5      S:  S9R                  5       nUSSS/0:X  d   eUR                  U S	S/S9R                  5       nUS[        [        S
5      5      S-  0:X  d   e[        U R                  5       5      nUR                  SS/S9R                  5       nUSS/0:X  d   eUR                  S	S/S9R                  5       nUS[        [        S
5      5      0:X  d   eg )Nr   rg   r"   rb   r.   r{   r/   r  r^      r#   )r   rD   rJ   rp   rB   rB  r   rE   rm   r4   r7   rP   )rX   r  rI  rY   s       r   	test_headr  |  s{     !,FRXX**2gnn*EEEE  !eW =GGIFeaS\!!!  !eW(*!(; ! ==FY[ eaV_$$$  $ @JJLFeT%(^a/0000G))+,H]]1ug].88:FeaS\!!!]]4%]1;;=FeT%(^,,,,r   c                    [        U R                  5       5      nSS/[        R                  " SS/5      4 H:  nUR	                  U5      R                  U5      nUR                  X#5      U:X  a  M:   e   [        R                  " [        5         UR                  U[        R                  " S/5      5        S S S 5        SS/[        R                  " SS/5      4 H8  nUR                  X5      UR	                  U 5      R                  U5      :X  a  M8   e   [        R                  " [        5         UR                  U [        R                  " S/5      5        S S S 5        g ! , (       d  f       N= f! , (       d  f       g = f)Nr"   r^   r#   re  r   )	r7   rP   rD   r   r   r   r  r   
IndexError)rX   r  rY   r   expecteds        r   	test_taker    s6   G))+,HFBHHaV,-!**8499'B""85AAA . 
z	"Hbhhsm4 
# FBHHaV,-""!/!8!8!A!F!Fw!OP 	P P . 
z	"GRXXrd^4 
#	" 
#	" 
#	"s   (E!0(E2!
E/2
F c                    [        U R                  5       5      nUR                  U5      S:X  d   eUR                  U[        R                  " S5      S:H  S9S:X  d   eUR                  U 5      S:X  d   eUR                  U [        R                  " S5      S:H  S9S:X  d   eUR                  U [        R                  " S5      S:  S9S:X  d   eUR                  U [        R                  " S5      S	:  S9S	:X  d   eg )
Nr#   rb   r   r  r"   r   r   r^   r   )r7   rP   r  r   rE   )rX   r  rY   s      r   test_count_rowsr    s   G))+,H$$X.!333$$%A- % /234 4 4 $$W-333$$)Q. % 0345 5 5 $$WRXXe_5I$JaOOO$$WRXXe_q5H$IQNNNr   c                      [         R                  [         R                  [         R                  /n U  H-  n[        R
                  " [        5         U" 5         S S S 5        M/     g ! , (       d  f       MA  = fr   )r   
FileFormatrp  Partitioningr  r   r!  )classesklasss     r   test_abstract_classesr    sJ    




G
 ]]9%G &% %%s   A$$
A3	c                  n   [         R                  " [         R                  " S[         R                  " 5       5      [         R                  " S[         R                  " 5       5      /5      n [
        R                  [
        R                  [
        R                  4 HA  nU" U 5      n[        U[
        R                  5      (       d   eX!" U 5      :X  d   eUS:w  a  MA   e   [         R                  " [         R                  " S[         R                  " 5       5      [         R                  " S[         R                  " 5       5      /5      n [
        R                  " U 5      n[        UR                  5      S:X  d   e[        S UR                   5       5      (       d   eUR                  S5      n[        U[
        R                  5      (       d   e[
        R                  " S5      S	:H  [
        R                  " S5      S
:H  -  nUR!                  U5      (       d   e["        R$                  " [         R&                  5         UR                  S5        S S S 5        UR                  S5      n[
        R                  " S5      S	:H  nUR!                  U5      (       d   eU[
        R                  " U SS9:w  d   e[         R                  " [         R                  " S[         R                  " 5       5      [         R                  " S[         R                  " 5       5      /5      n [
        R                  " U SS9n[        UR                  5      S:X  d   e[        S UR                   5       5      (       d   eUR                  S5      n[
        R                  " S5      [
        R(                  " S5      :H  [
        R                  " S5      [
        R(                  " S	5      :H  -  nUR!                  U5      (       d   eUR                  S5      n[
        R                  " S5      R+                  5       [
        R                  " S5      [
        R(                  " S	5      :H  -  nUR!                  U5      (       d   eS HA  n["        R$                  " [         R&                  5         UR                  U5        S S S 5        MC     U[
        R                  " U SS9:w  d   e[         R                  " [         R                  " S[         R                  " 5       5      [         R                  " S[         R                  " 5       5      /5      n [
        R                  " U 5      n[        UR                  5      S:X  d   e[        S UR                   5       5      (       d   eUR                  S5      n[        U[
        R                  5      (       d   e[
        R                  " S5      S	:H  [
        R                  " S5      S
:H  -  nUR!                  U5      (       d   e["        R$                  " [         R&                  5         UR                  S5        S S S 5        U[
        R                  " U SS9:w  d   e[         R                  " [         R                  " S[         R                  " 5       5      [         R                  " S[         R,                  " [         R.                  " 5       [         R0                  " 5       5      5      /5      n [
        R                  " U S[         R2                  " / SQ5      0S9nUR                  S   b   eUR                  S   R5                  5       / SQ:X  d   eU[
        R                  " U S S9:w  d   e[
        R                  " [         R                  " [         R                  " S[         R                  " 5       5      [         R                  " S[         R,                  " [         R.                  " 5       [         R0                  " 5       5      5      /5      S[         R2                  " / SQ5      0S9nUR                  S   b   eUR                  S   R5                  5       / SQ:X  d   e[         R6                  " [         R2                  " [9        S5      5      [         R2                  " S  [9        S5       5       5      [         R2                  " S!/S"-  S#/S"-  -   5      // S$QS%9n[         R                  " S&[         R0                  " 5       4/5      n[
        R                  [
        R                  [
        R                  4 H~  n[:        R<                  " 5        nU" U5      n[
        R>                  " XhS'US(9  [
        R@                  " US'US(9n	U	RC                  5       n
U
R!                  U5      (       d   e S S S 5        M     [:        R<                  " 5        n[
        R                  " U5      n[
        R>                  " XhS'US(9  S n	["        R$                  " [D        S)S*9   [
        R@                  " US'[G        S5      S(9n	S S S 5        U	b   e S S S 5        g ! , (       d  f       GN= f! , (       d  f       GM   = f! , (       d  f       GN;= f! , (       d  f       GM]  = f! , (       d  f       Ng= f! , (       d  f       g = f)+Nrb   rc   zother objectr   r   r{   c              3   (   #    U  H  oS L v   M
     g 7fr   r    .0r#  s     r   	<genexpr>$test_partitioning.<locals>.<genexpr>       <";QDy";   z/3/3.14/r^   gQ	@z/prefix/3/aaaz/3/nonesegment_encodingalphabetaxyz)null_fallbackc              3   (   #    U  H  oS L v   M
     g 7fr   r    r  s     r   r  r    r  r  z/alpha=0/beta=3/r   z/alpha=xyz/beta=3/)z/alpha=one/beta=2/z/alpha=one/z
/beta=two/otherc              3   (   #    U  H  oS L v   M
     g 7fr   r    r  s     r   r  r     r  r  z3_3.14_prefix_3_aaa_)firstsecondthirddictionariesr"      c              3   L   #    U  H  n[         R                   " 5       v   M     g 7fr   randomr  r  s     r   r  r  &       %Iy!fmmooy   "$r`   r   ra   r+  f2r   namesr   ipcr   r   z,Expected Partitioning or PartitioningFactoryr  )$rD   rB   rE   rG   rH   r   r   HivePartitioningFilenamePartitioningrR   r  r   r  allparse
Expressionr  r  r   rq  r  is_nullr   r   rI   r   rx  r   r4   tempfileTemporaryDirectorywrite_datasetrX   r   r^  int)rB   r  r   exprr  
shouldfailr   partitioning_schematempdir	load_backload_back_tables              r   test_partitioningr    s   YY

#


% F **B,?,?))+V},8888uV},,,~---+ YY
"((*%


% F ++F3L|(()Q...<,";";<<<<<j)DdBMM****!Q&288E?d+BCH;;x    	r	'?+ 
( e$Dxx A%H;;x    233FVTTTTYY
"((*%
$ F &&vUCL|(()Q...<,";";<<<<<01D	'	biil	*	&	RYYq\	)	+  ;;x    23D	'		"	"	$(8BIIaL(H	I  ;;x    I
]]2??+z* ,+ J 2..vWMMMMYY
"((*%


% F **62L|(()Q...<,";";<<<<<i(DdBMM****!Q&288E?d+BCH;;x    	r	'?+ 
( 2226FSSSSYY
"((*%
bggi=> F ++eRXX.J%KLL $$Q'///$$Q'113 8$ $ $ $233FNNNN**
		HHWbhhj)HHUBMM"'')RYY[AB
 	
 28889
L $$Q'///$$Q'113 8$ $ $ $ HH
rRXX%IuRy%II
#sebj()+ #E
 ))fbiik%:$;<**B,?,?))+((*g !45LU$)F

750<>I'002O"))%0000 +*+ 
	$	$	&'//0CD
 %L	B	]]:!OQ

75s1vNIQ     
'	&A 
(	'< ,+$ 
(	'L +*Q Q 
'	&sV   :ii*i0AjAj& j4j&
i
i-	0
i?
j	
j#	j&&
j4c                 .   [         R                  " [         R                  " S[         R                  " 5       5      [         R                  " S[         R                  " 5       5      /5      n[
        R                  " U5      [
        R                  " U5      [
        R                  " U5      [
        R                  " USS9[
        R                  " USS9[
        R                  " USSS9/nU H)  nU R                  U R                  U5      5      U:X  a  M)   e   g )Nrb   rc   r  r  r  )r  r  )rD   rB   rE   rG   rH   r   r   r  r  loadsdumps)pickle_modulerB   partsr   s       r   test_partitioning_picklingr  B  s    YY

#


% F
 	  (
F#
'
  &A
@
FV5QE ""=#6#6t#<=EEE r   z@flavor, expected_defined_partition, expected_undefined_partition))r  )zfoo=A/bar=ant%20bee r  r  )r   )z	A/ant beer  r  )r  )r  z
A_ant bee_)r  r  flavorexpected_defined_partitionexpected_undefined_partitionc                 2   [         R                  " S[         R                  " 5       4S[         R                  " 5       4/5      n[        [        U 5      " US9nUR                  [        R                  " S5      S:H  [        R                  " S5      S:H  -  5      U:X  d   eUR                  SR                  U5      5      R                  [        R                  " S5      S:H  [        R                  " S5      S:H  -  5      (       d   eUR                  [        R                  " S5      S:H  [        R                  " S5      S:H  -  [        R                  " S5      S:H  [        R                  " S5      S:H  -  -  5      U:X  d   eUR                  [        R                  " S5      S:H  [        R                  " S5      S:H  -  [        R                  " S5      S:H  [        R                  " S5      S:H  -  -  5      U:X  d   eU S:w  aU  [        R                  " [         R                  SS	9   UR                  [        R                  " S5      S:H  5        S S S 5        g UR                  [        R                  " S5      S:H  5      S
:X  d   eg ! , (       d  f       g = f)Nfoobarrg   zant beeAr   r  zDNo partition key for foo but a key was provided subsequently for barr  )zbar=ant%20beer  )rD   rB   rI   getattrr   r   pcrE   r  joinr  r  r   rq  )r  r  r  r  r   s        r    test_dataset_partitioning_formatr  T  s=    ))eRYY[%9E299;;O$PQ2v&.ABL 	RXXe_	9bhhuoQT>TUV%	&	&
 chh'ABCJJ	%C	BHHUOy$@A    	hhuo*rxx#/EF9,%C1GHJ	
 &		&	& 	hhuo*rxx#/EF9,%C1GHJ	
 (		(	( ## ]]OO(
 "((5/Y">A
 
 ""RXXe_	%AD I
 
 	
 

 
s   ')J
Jc                     [         R                  " [        R                  " / SQ/ SQS.5      5      n [         R                  " S5      n[         R                  " S5      nU R                  US-   X!-
  US-  UR                  S5      U-  S	.S
9n[        R                  " / SQ/ SQ/ SQ/ SQS	.5      nUR                  U5      (       d   eg )Nr"   r{   r^   )r{   r{   r{   r_   r`   ra   r"   r{   rH   )za+1zb-aza*2za/br.   r{   r^   r   )r"   r   r]  )r{   r      )      ?r1  g      ?)r   rX   rD   r   rE   r   castr  )rX   r`   ra   rI  r  s        r   $test_expression_arithmetic_operatorsr    s    jj		"BCDG
A
A1uu1uvvi 1$	' F xx H ==""""r   c                     S V s/ s H  n [         R                  " U 5      U :H  PM     sn u  pn[         R                  " U5      SS0:X  d   e[         R                  " U5      [         R                  " U5      :X  d   e[         R                  " X-  U-  5      S V s0 s H  o U _M     sn :X  d   e[         R                  " S5      S:  n[         R                  " U5      0 :X  d   e[         R                  " X-  5      SS0:X  d   e[         R                  " S5      R	                  5       n[         R                  " U5      SS 0:X  d   eg s  sn f s  sn f )Nabcr`   r   r^   )r   rE   get_partition_keys_get_partition_keysr  )fr`   ra   r   nopenulls         r   test_partition_keysr    s    )./Arxx{a/GA!  #Sz111  #r'='=a'@@@@  +e/De1e/DDDD88C=AD  &",,,  *sCj88888C=  "D  &3+555 0 0Es   #EEc                     [         R                  " 5       n [         R                  " SS/S9n[         R                  " SS9n[         R                  " [        R                  " 5       S9n[         R                  " [        R                  S9nU R
                  [        5       :X  d   eUR
                  SS1:X  d   eU R                  S:X  d   eUR                  S:X  d   eU R                  [        R                  " 5       :X  d   eUR                  [        R                  " 5       :X  d   eU R                  [        R                  L d   eUR                  [        R                  L d   eX :X  d   eX:w  d   eX:w  d   eX:w  d   eX:w  d   eS Ul        UR                  [        R                  " 5       :X  d   eX:X  d   e[        R                  " 5       Ul        UR                  [        R                  " 5       :X  d   eX:w  d   e[        R                  Ul
        UR                  [        R                  L d   eX@:X  d   e[        R                  Ul
        UR                  [        R                  L d   eX@:w  d   eg )	Nr`   ra   dictionary_columnsmscoerce_int96_timestamp_unitbinary_type	list_typens)r   ParquetReadOptionsrD   binary_viewLargeListTyper  r   r  r  binaryr  ListTypelarge_binary)opts1opts2opts3opts4opts5s        r   test_parquet_read_optionsr    s   !!#E!!c3Z@E!!dCE!!bnn.>?E!!B,<,<=E##su,,,##Sz111,,444,,444		+++ 0000??bkk)))??b.....>>>>>>>>>>E		+++>>)E 1111>>kkEO??bkk)))>>&&EO??b.....>>r   c                     [         R                  " 5       n [         R                  " S1S9n[         R                  " SS9n[         R                  " [        R                  " 5       S9n[         R                  " [        R                  S9nU R
                  [         R                  " 5       :X  d   eUR
                  [         R                  " S/S9:X  d   eUR
                  [         R                  " SS9:X  d   eUR
                  [         R                  " [        R                  " 5       S9:X  d   eUR
                  [         R                  " [        R                  S9:X  d   eg )Nr`   r  sr  r  r  )r   r   rD   r  r  read_optionsr  )pff1pff2pff3pff4pff5s        r   %test_parquet_file_format_read_optionsr    s"   !DC59DC@DBNN,<=D"*:*:;D 5 5 7777 5 5# OOOO 5 5$'!) ) ) ) 5 5NN$!& & & & 5 5""!$ $ $ $r   c                  <   [         R                  " 5       n [         R                  " SS9n[         R                  " SSS9n[         R                  " SSS9n[         R                  " SS	S
9n[         R                  " SS9n[        R                  " SSSS9n[         R                  " SUS9nU R                  SL d   eU R
                  S:X  d   e[        5       (       a  U R                  SL d   eU R                  S:X  d   eU R                  S:X  d   eU R                  SL d   eUR                  SL d   eUR
                  S:X  d   e[        5       (       a  UR                  SL d   eUR                  SL d   eUR
                  S:X  d   e[        5       (       a  UR                  SL d   eUR                  SL d   eUR
                  S:X  d   e[        5       (       a  UR                  SL d   eUR                  S:X  d   eUR                  S	:X  d   eUR                  SL d   e[        5       (       a  UR                  SL d   eUR                  U:X  d   eUR                  U R                  :w  d   eX :X  d   eX:w  d   eX:w  d   eX#:w  d   eX@:w  d   eXP:w  d   eXp:w  d   eg )N   buffer_sizei    T)r  use_buffered_streamF)r  
pre_bufferi@ i )thrift_string_size_limitthrift_container_size_limitpage_checksum_verificationr  )hole_size_limitrange_size_limitlazy)r  cache_optionsi @B )r   ParquetFragmentScanOptionsrD   CacheOptionsr  r  r   r  r  r  r  r  )r  r   r  r  r  opts6
cache_optsopts7s           r   test_parquet_scan_optionsr"    s   ))+E))d;E))t5E))eNE))!'$*-E ))#')EdDJ))TTE$$---%%%4'''))[888,,	999++u444$$---%%%4'''$$,,,%%%4'''$$---%%%5((())V333,,666++t3334'''*,,,%"5"5555>>>>>>>>>>>>>>r   c                    [         R                  " 5       [         R                  " 5       [         R                  " [        R                  R                  SSS95      [         R                  " [        R                  R                  SS/S9S9[         R                  " [        R                  R                  SSS	9S9[         R                  " 5       [         R                  " [        R                  R                  SS
S9S9[         R                  " [        R                  R                  SSS9S9/n UR                  [         R                  " 5       5        [        bb  UR                  [         R                  " 5       [         R                  " S1S9[         R                  " SS9[         R                  " SSSSS9/5        U H)  nU R                  U R!                  U5      5      U:X  a  M)   e   g ! [         a     Nf = f)N	T)	delimiterignore_empty_linesr^   r  )	skip_rowsrw  r  i   )r'  
block_sizeignorenewlines_in_valuesunexpected_field_behavior)parse_optionsF   r   r)  r`   r  )r  r  {   i  )r  r  r  r  )r   r,  CsvFileFormatrD   csvParseOptionsReadOptionsJsonFileFormatjsonr5   OrcFileFormatImportErrorrq   extendr   r  r  )r  formatsr"  s      r   test_file_format_picklingr<  ,  s   


,,t@D - F 	G
bff&8&8ug '9 '/ 	0
bff&8&8E '9 '+ 	,

''..$IQ / S	T 	rww':':" (; (. 	/G r'')* 
~  "  SE:  T:  $( ),,/		

 
	 ""=#6#6{#CDSSS !  s   .$G+ +
G87G8c                    [         R                  " 5       [         R                  " [        R                  R	                  SS9S9[         R                  " [        R                  R                  SS9S9[         R                  " 5       [         R                  " [        R                  R                  SSS	95      [         R                  " [        R                  R                  SS
S9S9/n[        b7  UR                  [         R                  " SS9[         R                  " SS9/5        U H)  nU R                  U R                  U5      5      U:X  a  M)   e   g )NT)strings_can_be_nullconvert_options   r)  r(  Ferrorr+  i   r0  r  r  )r  )r   CsvFragmentScanOptionsrD   r3  ConvertOptionsr5  JsonFragmentScanOptionsr7  r4  rq   r:  r  r  r  )r  r   options      r   #test_fragment_scan_options_picklingrH  S  s    
!!#
!!FF11d1K	M
!!++u+=	?
""$
""GG  E;B ! D	E 	"",,#,N	PG 
~))d;))T:
 	
 ""=#6#6v#>?6III r   paths_or_selectorr   r   r  r  r  c                 ,   [         R                  " [         R                  " S1S9US9n[         R                  " S5      n[         R                  " [
        R                  " [
        R                  " S[
        R                  " 5       5      [
        R                  " S[
        R                  " 5       5      /5      5      Ul
        UR                  S:X  d   eUR                  SS/:X  d   eUR                  S	L d   e[         R                  " XX45      nUR                  5       nUR                  5       R!                  [
        R                  " [
        R                  " S
[
        R"                  " 5       5      [
        R                  " S[
        R$                  " 5       5      [
        R                  " S[
        R&                  " [
        R                  " 5       [
        R                  " 5       5      5      [
        R                  " S[
        R"                  " 5       5      [
        R                  " S[
        R(                  " [
        R"                  " 5       [
        R                  " 5       S.5      5      [
        R                  " S[
        R                  " 5       5      [
        R                  " S[
        R                  " 5       5      /5      S	S9(       d   e[+        UR-                  5       [.        5      (       d   e[+        UR1                  U5      [         R2                  5      (       d   eUR4                  R!                  [         R6                  " S5      5      (       d   eUR1                  5       n[+        U[         R2                  5      (       d   eUR9                  5       n[
        R:                  " / SQ[
        R"                  " 5       S9n	[
        R:                  " / SQ[
        R$                  " 5       S9n
[
        R<                  R?                  [
        R:                  " / SQ[
        R                  " 5       S9[
        R:                  " SRA                  5       [
        R                  " 5       S95      n[
        R:                  " [C        S5       Vs/ s H  nUS-  [E        US-  5      S.PM     sn5      nURG                  5       n[I        USS/SS/5       GHp  u  u  nnnn[
        R:                  " U/S-  [
        R                  " 5       S9n[
        R:                  " U/S-  [
        R                  " 5       S9n[
        R:                  " US-
  /S-  [
        R"                  " 5       S9nURJ                  c   eURL                  S:X  d   eUS   R!                  U	5      (       d   eUS   R!                  U
5      (       d   eUS   R!                  U5      (       d   eUS   R!                  U5      (       d   eUS   R!                  U5      (       d   eUS   R!                  U5      (       d   eUS   R!                  U5      (       a  GMq   e   URO                  5       n[+        U[
        RP                  5      (       d   e[S        U5      S:X  d   eURL                  S:X  d   eg s  snf )Nrd   r  )r  r  r   r   r   .r  Frb   rc   re   rf   r_   check_metadataTr0  r|   z	0 1 2 3 4r#   r^   r"   r{   r2  r3  re  r   r   r  r   )*r   r   r  r   r   rD   rB   rE   r   rI   r   partition_base_dirselector_ignore_prefixesexclude_invalid_filesr   rS  r  rG   rH   r   rf   rR   inspect_schemasrm   r   r  r  r  r   r   r   r   splitr4   rd   r>  r  r  rf  r   rJ   r   )rs   rI  r  r   r   r   inspected_schemarX   r   rF  rG  expected_strr?   expected_structiteratorry   rY   r   r   expected_groupexpected_keyexpected_constr   s                          r   test_filesystem_factoryrZ  l  s    !!**ugFF
 ))(3G33
		HHWbhhj)HHUBIIK(
 	G %%111++Sz999((E111))6G (??##BII

#


%
bhhj"))+>?
"((*%
299288:+-99;&8 9 	:
"((*%
		$	/ 	%  $ 	 	 	 g--/6666gnn%56**, , , ,!!((49999nnGgr334444ooG88O"((*=L88O"**,?L%%11
rxxz2
""$299;7L hh).q 3)1A '(!e#a!e* =)1 3 4O##%H),X1vu~)N%5#5'A+BHHJ?xx			<519+/
C,,888  A%%%Qx|,,,,Qx|,,,,Qx|,,,,Qx~....Qx////Qx~....Qx|,,,, *O EeRXX&&&&u:!!!+ 3s   Zc                    [         R                  " 5       n[         R                  " SU US9nUR                   H  nUR	                  X05      nUR
                  S/:X  d   eUR	                  X0S/S9nXE4 H\  n[        U[         R                  5      (       d   eUR                  U:X  d   e[        UR                  [        U 5      5      (       a  M\   e   UR
                  S/:X  a  M   e   g )N/plainr
  r   r   r  )r   r   rX   r  r  r  rR   r  rv   r
  r}   )r   parquet_formatrX   rv   rY   row_group_fragmentr  s          r   test_make_fragmentra    s    ))+Njjm .0G !//D""qc)))+99$FGS : J/Aa!7!7888866T>!>allD,?@@@@ 0 ",,333 r   c           
         U u  pp4pVpx[         R                  " 5       n	U/n
U
 Vs/ s H  nU	R                  X#5      PM     nn[         R                  " XUR                  US9nUR                  5       nUR                  U5      (       d   eUR                   Vs/ s H'  oR                  R                  U5      R                  PM)     nn[        X5       VVs/ s H  u  nnU	R                  X#US9PM     nnn[         R                  " UXR                  US9nUR                  5       nUR                  U5      (       d   eU
 Vs/ s H  nSPM     nn[        U
U5       VVs/ s H  u  nnU	R                  X#US9PM     nnn[         R                  " UXR                  US9n[        R                  " [        R                  R                   SS9   UR                  5       nSSS5        U
 Vs/ s H  nSPM     nn[        U
U5       VVs/ s H  u  nnU	R                  X#US9PM     nnn[         R                  " UXR                  US9n[        R                  " ["        SS9   UR                  5       nSSS5        gs  snf s  snf s  snnf s  snf s  snnf ! , (       d  f       N= fs  snf s  snnf ! , (       d  f       g= f)	z
Test passing file_size to make_fragment. Not all FS implementations make use
of the file size (by implementing an OpenInputFile that takes a FileInfo), but
s3 does, which is why it's used here.
)r   rB   r
  )	file_sizer"   zParquet file size is 1 bytesr  Nr  zHTTP status 416)r   r   r  r  rB   r   r  r  r
  get_file_infosizer  r  r   r   librq  OSError)s3_example_simpler   rv   rh   urihostport
access_key
secret_keyr"  r   r$  rX   tblr#  
sizes_truere  fragments_with_sizedataset_with_sizesizes_toosmallsizes_toolarges                        r   test_make_fragment_with_sizert    s    @Q<E$j&&(KFE #$"T **44"  $""ellrG 


C::e ELMMRMq$$2215::MJR-0-CE-CztT '44T4N-C  E,,KQS 


C::e %**EDaEN*-0-GI-GztT '44T4N-G  I ,,KQS 
w{{//7U	V!**, 
W +00%$g%N0-0-GI-GztT '44T4N-G  I ,,KQS
 
w&7	8!**, 
9	8S$ SE +I 
W	V 1I 
9	8sA   I8.I=JJJJ(J$J)J/
J!/
J=c                 \   [         R                  " S5      n[        R                  " UR	                  S5      5      n[
        R                  " 5       nUR                  U5      n[        UR                  5       [        R                  5      (       d   e[        R                  " / SQ/ SQ/ SQ// SQS9nU R                  U5      R                  U5      (       d   eUR                  UR                  U5      5      nU R                  U5      R                  UR                  5       5      (       d   eg )NzT
        alpha,num,animal
        a,12,dog
        b,11,cat
        c,10,rabbit
    utf-8r`   ra   r         r   dogcatrabbitr  numanimalr  )textwrapdedentrD   	py_bufferencoder   r2  r  rR   rQ   BufferReaderr   r   r  r  r  )r  r  contentbuffer
csv_formatrY   r  pickleds           r   "test_make_csv_fragment_from_bufferr  
  s    oo  	G \\'..12F!!#J''/H hmmor7777xx%13 9:H ""8,33H====!!-"5"5h"?@G""7+2283D3D3FGGGGr   c                 4   Sn[         R                  " UR                  S5      5      n[        R                  " 5       nUR                  U5      n[        UR                  5       [         R                  5      (       d   e[         R                  " / SQ/ SQ/ SQ// SQS9nU R                  U5      R                  U5      (       d   eUR                  UR                  U5      5      nU R                  U5      R                  UR                  5       5      (       d   eg )Nz{"alpha" : "a", "num": 12, "animal" : "dog"}
{"alpha" : "b", "num": 11, "animal" : "cat"}
{"alpha" : "c", "num": 10, "animal" : "rabbit"}
rv  rw  rx  r{  r  r  )rD   r  r  r   r6  r  rR   rQ   r  r   r   r  r  r  )r  r  r  r  json_formatrY   r  r  s           r   #test_make_json_fragment_from_bufferr  #  s    <G \\'..12F##%K((0H hmmor7777xx%13 9:H ""8,33H====!!-"5"5h"?@G""7+2283D3D3FGGGGr   c                    [         R                  " / SQ5      [         R                  " / SQ5      [         R                  " / SQ5      /nUS   R                  5       US   US   R                  5       /n[        R                  " [        R
                  " SS/S	9S
SS9nU[        R                  " 5       4X44/nU GH  u  p&[         R                  " U/ SQS9n[         R                  " 5       n[        R                  " Xx5        UR                  5       n	UR                  U	5      n
U R                  U
5      R                  U5      (       d   eUR                  UR                  U
5      5      nU R                  U5      R                  U5      (       d   e[         R                   " U	5      nUR                  U5      n
U
R#                  5       n[%        U[         R&                  5      (       d   eUR(                  (       d   e[        R*                  " U
R#                  5       5      R-                  5       R                  U5      (       a  GM   e   g )Nrw  rx  r{  r   r"   r{   r  r  r  Tr  )r  r  r  r  r  )rD   r   dictionary_encoder   r   r  r   BufferOutputStreamrq   rr   getvaluer  r   r  r  r  r  rQ   rR   rS   rV   ParquetFileread)r  r  arraysdictionary_arraysdictionary_formatcasesformat_r   rw   r  rY   r  	file_likeopened_files                 r   &test_make_parquet_fragment_from_bufferr  9  s    	!

)*F 	q	##%q	q	##%
 ,,** '2
 ! 
%%'(	.E !'AB##%
u"((0&&x077>>>>%%m&9&9(&CD&&w/66u==== OOF+	((3mmo+r}}5555####~~hmmo.335<<UCCCC' !r   c                     [         R                  " [        S5      S/S-  S/S-  S/S-  -   // SQS9n[        U S-  5      n[        R
                  " X4S	/US
9  [        R                  " USSUS9nX54$ )Nrb  r"   r`   r   ra   r  r  test_parquet_datasetr   )partition_cols
chunk_sizer   r   )r   r   r
  )rD   r   r4   rd   rq   write_to_datasetr   rX   )r  r  r
  r   rv   rX   s         r   _create_dataset_for_fragmentsr  g  s    HH	qA37SEAI	12"E
 w//0D(.xJHjjYV
G >r   c                    [        U 5      u  p#[        UR                  5       5      n[        U5      S:X  d   eUS   nSS/nUR                  R
                  U:X  d   eUR                  R                  UR                  UR                  5      UR                  :X  d   eUR                  R                  [        R                  " S5      S:H  5      (       d   eUR                  U5      nUR                  U:X  d   eUR                  UR!                  S5      R#                  SS5      5      (       d   eUR                  XSR$                  S9nUR                  / S	Q:X  d   eUR                  UR#                  SS5      5      (       d   eUR                  UR$                  R'                  S5      :X  d   eUR                  XSR$                  [        R                  " S5      S:  S
9nUR                  / S	Q:X  d   eg )Nr{   r   r+  r  r   r`   r   rg   r  )rB   r  )r  rm   rP   r   physical_schemar  r   rS  rv   r
  r  r  r   rE   r   rw  remove_columnslicerB   remove)r  r  r   rX   r$  r  physical_namesrI  s           r   test_fragmentsr  y  s   27;NE W**,-Iy>Q!AD\N""n44488AFFALL1Q5F5FFFF!!((&)9S)@AAAA $$Q'F.000==,,Q/55a;<<<< $$Q~~$>F"6666==Q*++++ 4 4Q 7777 $$	..$!); % =F"6666r   c                    [         R                  " [        S5      S/S-  S/S-  -   /SS/S9n[        U S-  5      n[        R
                  " XS/S	9  [        R                  " [         R                  " S
/5      SS9n[        R                  " USUS9nUR                  [        R                  " S5      S:  S9n[        [        U5      5      S:X  d   eg )Nrb  r"   r   r{   colr   r  r  r  )r   r   r   r  r   r  r  )rD   r   r4   rd   rq   r  r   r   rB   rX   rP   rE   r   rm   )r  r   rv   r   rX   r$  s         r   test_fragments_implicit_castr    s     HHeAha1#' 125&/JEw//0DVH=??299&6%78HDjjidCG%%RXXf-=-B%CItI1$$$r   c                   ^
 [        U 5      u  m
n SU
4S jjn[        UR                  5       5      S   nUR                  nUR	                  UR                  U5      5      nUR                  U5      UR                  U5      :X  d   eUR                  UR                  UR                  UR                  S9nUR                  U5      R                  UR                  U5      5      (       d   eU" US5        UR                  UR                  UR                  UR                  S9nU" US[        R                  " S5      S:  S9  UR                  UR                  UR                  UR                  S9nU" USS/[        R                  " S5      S	:  S
9  UR                  UR                  UR                  UR                  S9nU" US[        R                  " S5      S:H  S9  SUR                  R                  SSS5      -   n	[         R"                  " [$        U	S9   UR                  UR                  UR                  UR                  S9nUR                  U[        R                  " S5      S:H  S9  S S S 5        g ! , (       d  f       g = f)Nc                    > U R                  TR                  X#S9nU(       a  UOTR                  nUR                  U:X  d   eTR                  " U6 R	                  U5      nUR                  U5      (       d   eg )NrB   r/   r  )r   rB   rw  r  selectr  )rY   	row_slicer/   r  actualrw  r  r   s          r   assert_yields_projected;test_fragments_reconstruct.<locals>.assert_yields_projected  ss    ""<< # A")wu/A/A""l222;;	*11,?}}X&&&&r   r   )r  )r   r   )r   r{   r+  r{   r  r8  r  r   r`   z&No match for FieldRef.Name\(part\) in Fr  NN)r  rm   rP   r   r  r  r   r  rv   r
  r  r  r   rE   r  	to_stringr  r   r^  )r  r  r  rX   r  rY   r_  pickled_fragmentnew_fragmentpatternr   s             @r   test_fragments_reconstructr    su   27;NE7 6:' G))+,Q/H__N %**=+>+>x+HI""+44X>? ? ? "//x**%:: 0 <L ""<077)+ + + +L&1 "//x**%:: 0 <L L&$!9KL "//x**%:: 0 <L L&%)F288D>C3GI "//x**%:: 0 <L L&#%88F#3s#:<
 9''11%FGG	z	1%33MM8..!)!>!> 4 @ 	RXXf5E5LM	 
2	1	1s   AI::
Jc                    [        U SS9u  p#[        UR                  5       5      S   n[        UR                  5       5      n[	        U5      UR
                  s=:X  a  S:X  d   e   eUR                  US   UR                  S9nUR                  / SQ:X  d   e[	        U5      S:X  d   eUR                  UR                  SS5      5      (       d   eUS   R                  c   eUS   R
                  S:X  d   eUS   R                  S   R                  SSS.SSS.S.:X  d   e[        UR                  [        R                  " S	5      S:  S
95      S   n[        UR                  [        R                  " S	5      S:  5      5      n[	        U5      S:X  d   eUR                  US   [        R                  " S	5      S:  S
9n[	        U5      S:X  d   eg )Nr{   r  r   rg   r  r"   minmaxr+  r  r+  r  )r  rm   rP   r  r   r  r   rB   rw  r  r  r  
statisticsr   rE   )r  r  r   rX   rY   r(  rI  s          r   !test_fragments_parquet_row_groupsr    s   27qINEG))+,Q/H x::<="#x'>'>C!CCCCC$$Aw~~ % 7F"6666v;!==Q*++++q!,,888q!00A555q!,,Q/::""?   
 G))$!1C)DEaHHx::288D>A;MNO"#q((($$Arxx~'9 % ;Fv;!r   c                    [         R                  " S[        S5      05      n[        R                  " XS-  SS9  [
        R                  " U S-  SS9n[        UR                  5       5      S   nUR                  R                  UR                  UR                  S	S
/S9nUR                  S:X  d   eUR                  5         UR                  S:X  d   e[        UR                   5      S:X  d   eg )Nr`   rb  test.parquetr{   row_group_sizer   r  r   r"   r^   r^  )rD   r   r4   rq   rr   r   rX   rm   rP   r   r  rv   r
  r  ensure_complete_metadatar   r  )r  r   rX   original_fragmentrY   s        r   %test_fragments_parquet_num_row_groupsr    s    HHc58_%ENN5N21Ejj>1)DGW2245a8 !''55 1 < <q6 6 H ""a'''%%'""a'''x""#q(((r   c                    [         R                  " [        SS/SS/S95      nUS   R                  S5      US'   [        R
                  " [        R                  " U5      U S-  5        S	S KJ	n  UR                  " U S-  5      nUR                  XCR                  " S5      S:H  S
9nUR                  S	   UR                  5       :H  R                  5       R                  5       (       d   eg )Nr`   ra   r"   r{   )col1col2r  categoryztest_filter_dictionary.parquetr   r  )r8   r9   dictastyperq   rr   rD   r   pyarrow.datasetrX   r   rE   r   	to_pandasr  )r  r  rM   r   rX   rI  s         r   ,test_fragments_parquet_row_groups_dictionaryr    s     
dc
!Q8	9BF"":.BvJNN288B<+K!KL jj#CCDG$$WXXf5E5L$MFGGAJ&**,,113779999r   c                     Uu  p4[        U SUS9u  pV[        UR                  5       5      S   nU" UR                  /5         UR	                  5         S S S 5        UR
                  SS/:X  d   eU" / 5         UR	                  5         S S S 5        [        UR                  [        R                  5      (       d   eUR                  R                  UR                  UR                  SS/S9nUR
                  UR
                  :X  d   eUR	                  5         UR
                  S   n	U	R                  S:X  d   eU	R                  S:X  d   eU	R                  c   eUR!                  UR#                  U5      5      n
U" UR                  /5         U
R
                  SS/:X  d   eU
R
                  S   n	U	R                  S:X  d   eU	R                  c   e S S S 5        g ! , (       d  f       GN= f! , (       d  f       GNm= f! , (       d  f       g = f)Nr{   r  r
  r   r"   r^  )r  rm   rP   rv   r  r  rR   metadatarq   FileMetaDatar   r  r
  idr   r  r  r  )r  r   r  rh   r   r  rX   rY   r  	row_groupr  s              r   &test_fragments_parquet_ensure_metadatar  $  s   &B.A"JA G))+,Q/H 
x}}o	&))+ 
'1a&((( 
b	))+ 
 h''9999 ??00x**1v 1 L ""h&9&9999 ))+''*I<<1"""+++ %**=+>+>|+LM	x}}o	&**q!f444$//2	||q   ##///	 
'	&3 
'	&
 
	( 
'	&s%   G6G-AG?
G*-
G<?
Hc                 b   Uu  p4[        XS9u  pV[        UR                  5       5      S   nU" / 5         UR                  UR	                  U5      5      nS S S 5        U" WR
                  /5         UR                  n	S S S 5        W	S/:X  d   eg ! , (       d  f       N@= f! , (       d  f       N)= f)Nr
  r"   r   )r  rm   rP   r  r  rv   r  )
r  r   r  rh   r   r  rX   rY   r  r  s
             r   )test_fragments_parquet_pickle_no_metadatar  M  s     'B.wFJAG))+,Q/H 
b	(..}/B/B8/LM 
 
',,-	.%00
 
/! 
	 
/	.s   !B0B 
B 
B.c                 h   [         R                  " [         R                  " / SQ[         R                  " 5       5      [         R                  " / SQ[         R                  " 5       5      [         R                  " / SQ[         R
                  " 5       5      [         R                  " / SQ[         R                  " 5       5      [         R                  " / SQ[         R                  " 5       5      [         R                  " / SQ[         R                  " 5       5      [         R                  " / SQ[         R                  " 5       5      [         R                  " / SQ[         R                  " 5       5      [         R                  " / SQ[         R                  " 5       5      [         R                  " / SQ[         R                  " 5       5      [         R                  " / SQ[         R                  " 5       5      [         R                  " / SQ[         R                  " 5       5      [         R                  " / SQ[         R                  " 5       5      [         R                  " / SQ[         R                   " S5      5      [         R                  " / SQ[         R                   " S5      5      [         R                  " / SQ[         R                   " S5      5      [         R                  " / SQ[         R"                  " 5       5      [         R                  " / SQ[         R$                  " 5       5      [         R                  " / SQ[         R&                  " S5      5      [         R                  " / SQ[         R(                  " S5      5      // S	QS
9n[+        U S-  5      n[,        R.                  " X#US9  U[0        R2                  " USSS94$ )N)TNF)r"   r   *   )r1  g      $@      E@)r`   Nzr  r  us)r"   r   l    jt )booleanr   uint8r   r   r   uint32rG   uint64r6   doubleutf8r  ts[s]ts[ms]ts[us]rF   date64time32time64r  test_parquet_dataset_all_typesr  r   r   r  )rD   r   r   bool_r   r  r   r   r   r  rG   r  float32rH   r  r  	timestamprF   r  r  r  rd   rq   r  r   rX   )r  r  r   rv   s       r   _create_dataset_all_typesr  _  s   HHHH("((*5HH["''),HH["((*-HH["((*-HH["))+.HH["((*-HH["))+.HH["((*-HH["))+.HH&

5HH&

5HH%rwwy1HH%ryy{3HH[",,s"34HH[",,t"45HH[",,t"45HH["))+.HH("))+6HH["))C.1HH["))D/2)	
,
/-E^ w99:D 
;"**T)&IIIr   c                 P  ^ [        U 5      u  p[        UR                  5       5      S   nSS KmU4S jnU4S jnU4S jnTR                  nTR
                  n[        UR                  5       5      n	U	S   R                  c   eU	S   R                  S   n
U
R                  S:X  d   eU
R                  S:  d   eU
R                  0 SSS	S
._SSSS
._SSSS
._SSSS
._SSSS
._SSSS
._SSSS
._SSSS
._SSSS
._SSSS
._SSSS
._SSSS
._SSSS
._SU" S5      U" S5      S
._S U" S5      U" S5      S
._S!U" S5      U" S5      S
._S"U" S#SS$5      U" S#S$S%5      S
._U" S#SS5      U" S#S$S&5      S
.U" SSS5      U" SSS5      S
.U" SSSS5      U" SSSS5      S
.S'.E:X  d   eg )(Nr   c                 2   > TR                   " SSSSSU 5      $ N  r"   r   r0   r#  r0   s    r   dt_s.test_parquet_fragment_statistics.<locals>.dt_s  s    ))$1aA>>r   c           
      :   > TR                   " SSSSSSU S-  5      $ )Nr  r"   r   r   r  r  s    r   dt_ms/test_parquet_fragment_statistics.<locals>.dt_ms  s#    **4Aq!Q$GGr   c           	      4   > TR                   " SSSSSSU 5      $ r  r  r  s    r   dt_us/test_parquet_fragment_statistics.<locals>.dt_us  s    **4Aq!QBBr   r^   r   r  FTr  r   r"   r  r  r   r   r   r  rG   r  r6   r1  r  r  r  r`   r  r     a   zr  r  r  rF   r  r{   ry     )r  r  r  )r  rm   rP   r0   r*   timer  r  r   total_byte_sizer  )r  r   rX   rY   r  r  r  r*   r  r(  r  r0   s              @r    test_parquet_fragment_statisticsr    sl    /w7NEG))+,Q/H>GB==D==D x::<=q!,,888#A&11!4I"""$$t+++ $5.$"%$ 	2&$ 	2&	$
 	!B'$ 	2&$ 	!B'$ 	2&$ 	!B'$ 	T*$ 	#d+$ 	C($ 	$t,$ 	ab2$ 	%(595$  	%(595!$" 	$tQ*4a3DE#$$ tQ*4a3DEq!Q-Q2?q!Q*41a3DE)$   r   c                 6   [         R                  " / SQ/ SQS.5      n[        R                  " XS-  SS9  [        R
                  " U S-  SS9n[        UR                  5       5      S	   R                  5       nUS
   R                  S	   R                  0 :X  d   eg )N)r   r"   NN)r`   ra   NNr_   r  r{   r  r   r  r   r"   )rD   r   rq   rr   r   rX   rm   rP   r  r  r  )r  r   rX   r$  s       r   &test_parquet_fragment_statistics_nullsr
    s    HH-4JKLENN5N21Ejj>1)DGW**,-a0CCEIQ<""1%00B666r   c                 2   [         R                  " / SQ/ SQS.5      S S nUR                  U S-  SS9  [        R                  " U S-  SS	9n[        UR                  5       5      S   R                  5       nUS   R                  S   R                  0 :X  d   eg )
N)r`   ra   ra   r   r#   r  r_   r   r  r   enginer   r  )
r8   r9   
to_parquetr   rX   rm   rP   r  r  r  )r  rM   rX   r$  s       r   'test_parquet_empty_row_group_statisticsr    s     
O)<	=bq	ABMM'N*9M=jj>1)DGW**,-a0CCEIQ<""1%00B666r   c                    [        U SS9u  p[        UR                  5       5      S   nUR                  R	                  [
        R                  " S5      S:H  5      (       d   e[        UR                  [
        R                  " S5      S:H  UR                  S95      n[        U5      S:X  d   e[        UR                  [
        R                  " S5      S:H  UR                  S95      n[        U5      S:X  d   eg )Nr{   r  r   r   r`   r  rB   ra   )
r  rm   rP   r  r  r   rE   r  rB   r   )r  r   rX   rY   r(  s        r   +test_fragments_parquet_row_groups_predicater    s    27qINEG))+,Q/H((//0@C0GHHHH ##288F+;s+B+2>> 	$ 	;< "#q((( ##288F+;s+B+2>> 	$ 	;< "#q(((r   c                     [        U SS9u  p4[        UR                  5       5      S   nUR                  n[        UR	                  5       5      nUR                  UR                  U5      5      nUR                  U5      UR                  U5      :X  d   eUR                  UR                  UR                  UR                  S/S9n	UR                  U	5      n
U
R                  UR                  US   5      5      (       d   eUR                  UR                  UR                  UR                  S1S9n	UR                  XR                  SS/[        R                  " S5      S:  S	9n
U
R                   SS/:X  d   e[#        U
5      S:X  d   eUR                  UR                  UR                  UR                  S1S9n	[$        R&                  " [(        S
S9   UR                  U	5        S S S 5        g ! , (       d  f       g = f)Nr{   r  r   )r  r  r"   r+  r   r^   r  zreferences row group 2r  )r  rm   rP   r   r  r  r  r   r  rv   r
  r  r  rB   r   rE   rw  r   r  r   r  )r  r  r  r   rX   rY   r_  r(  r  r  rI  s              r   -test_fragments_parquet_row_groups_reconstructr    s    37qINEG))+,Q/H__Nx::<= %**=+>+>x+HI""+44X>? ? ? "//x**%::3 0 L $$\2F==001DQ1GHIIII "//x**%::3 0 L $$\\D&>xx~! % %F 4.000v;! "//x**%::3 0 L 
z)A	B- 
C	B	Bs   G//
G=c                    Uu  p4[        U SUS9u  pV[        UR                  5       5      S   nUR                  SS/S9nU" / 5         UR                  S:X  d   eUR
                  SS/:X  d   eUR
                  S   R                  c   e S S S 5        UR                  U5      n	U	R                  5       SS/SS/S.:X  d   eUR                  / S9nUR                  S:X  d   eUR
                  / :X  d   eUR                  XR                  S9n	U	R                  S:X  d   eU	R                  US S 5      (       d   eg ! , (       d  f       N= f)	Nr"   r  r   r^   row_group_idsr{   r  rg   )r  rm   rP   subsetr  r  r  r   rB  rB   r   r  
r  r   r  rh   r   r   rX   rY   subfragrI  s
             r   !test_fragments_parquet_subset_idsr    sa    'B27q>@BNEG))+,Q/H ooQFo3G	b	%%***!!aV+++!!!$//;;; 
 $$W-FAq!f!==== ooBo/G!!Q&&&###$$W^^$DF??a==r#### 
	s   AD;;
E	c                    Uu  p4[        U SUS9u  pV[        UR                  5       5      S   nUR                  [        R
                  " S5      S:  5      nU" / 5         UR                  S:X  d   e[        UR                  5      S:X  d   eUR                  S   R                  c   e S S S 5        UR                  U5      n	U	R                  5       / SQ/ SQS.:X  d   eUR                  [        R
                  " S5      S	:  5      nUR                  S:X  d   eUR                  / :X  d   eUR                  XR                  S
9n	U	R                  S:X  d   eU	R                  US S 5      (       d   eUR                  [        R
                  " S5      S:H  UR                  S
9nUR                  S:X  d   eg ! , (       d  f       GN= f)Nr"   r  r   r+  r^   r  )r"   r"   r"   r  r#   rg   r   r`   r   )r  rm   rP   r  r   rE   r  r   r  r  r   rB  rB   r   r  r  s
             r   $test_fragments_parquet_subset_filterr  7  s    'B27q>@BNEG))+,Q/H oobhhtn12G	b	%%***7%%&!+++!!!$//;;; 
 $$W-F	!CCCC oobhhtnq01G!!Q&&&###$$W^^$DF??a==r#### oobhhv.#5gnnoMG!!Q&&&' 
	s   A
F55
Gc                    [        U SS9u  p[        UR                  5       5      S   n[        R                  " [
        5         UR                  [        R                  " S5      S:  SS/S9  S S S 5        [        R                  " [
        5         UR                  5         S S S 5        g ! , (       d  f       NB= f! , (       d  f       g = f)Nr"   r  r   r+  r{   r  )	r  rm   rP   r  r   r^  r  r   rE   )r  r  rX   rY   s       r   %test_fragments_parquet_subset_invalidr   W  s    .w1EJAG))+,Q/H 
z	"!+Aq6B 
# 
z	" 
#	" 
#	" 
#	"s   *B)B:)
B7:
Cc                 d   [         R                  " / SQ5      n[         R                  " / SQ5      n[         R                  " / SQ5      n[         R                  R                  X#/SS/S9n[         R                  R                  X/SS/S9n[         R                  " S	U05      n[
        R                  " X`S
-  SS9  [        R                  " U S
-  SS9n[        UR                  5       5      S   nUR                  S:X  d   eUR                  [        R                  " S	S5      S:  5      n	U	R                  S:X  d   eUR                  [        R                  " S	S5      S:  5      n	U	R                  S:X  d   eUR                  [        R                  " S	SS5      S:  5      n	U	R                  S:X  d   eUR                  [        R                  " S	SS5      S:*  5      n	U	R                  S:X  d   e[        R                  " [         R                   SS9   UR                  [        R                  " S	S5      S:  5        S S S 5        [        R                  " ["        SS9   UR                  [        R                  " S	S5      S:  5        S S S 5        g ! , (       d  f       NZ= f! , (       d  f       g = f)N)r   r"   r{   r^   )皙?皙?333333?皙?r"   r{   r^   r   f21f22r  r+  r  r  zdata_struct.parquetr{   r  r   r  r   r"   r#   zNo match for FieldRef.Nestedr  f3z)Function 'greater' has no kernel matching)rD   r   StructArrayr   r   rq   rr   r   rX   rm   rP   r  r  rE   r  r   rq  NotImplementedError)
r  r+  r'  r(  r  
struct_colr   rX   rY   r  s
             r   0test_fragments_parquet_subset_with_nested_fieldsr-  d  s*    
,	B
(('
(C
((<
 C		#	#SJuen	#	EB++RHT4L+IJHHeZ()ENN5$99!Ljj#88KGG))+,Q/H""a'''oobhhud3a78G!!Q&&&oobhhud3a78G!!Q&&&oobhhudE:Q>?G!!Q&&&oobhhudE:a?@G!!Q&&& 
r.L	M-12 
N 
#N
 	-12
 
	 
N	M
 
s   *J*J!
J!
J/c                 ~   [        UR                  5       5      S   n[        U5      S:X  d  [        U5      S:X  d   e[        U 5      u  p4[        R
                  " USS9n[        UR                  5       5      S   n[        U5      SUR                  R                  [        U5      5       S3:X  d   eU S-  n[        R                  R                  X45        [        R
                  " US	S9n[        UR                  5       5      S   n[        U5      S
UR                  R                  [        U5      5       S3:X  d   eg )Nr   zb<pyarrow.dataset.ParquetFileFragment path=subdir/1/xxx/file0.parquet partition=[key=xxx, group=1]>zb<pyarrow.dataset.ParquetFileFragment path=subdir/1/xxx/file0.parquet partition=[group=1, key=xxx]>r   r  z*<pyarrow.dataset.ParquetFileFragment path=>data.featherfeatherz,<pyarrow.dataset.FileFragment type=ipc path=)rm   rP   repr_create_single_filer   rX   r
  r   rd   rD   r1  write_feather)r  rX   rY   r   rv   s        r   test_fragments_reprr5    s>    G))+,Q/H 	X	(	( 	X	(	(	( &g.KEjji0GG))+,Q/HX4,,SY7
8	;	;	; ^#DJJU)jji0GG))+,Q/HX6,,SY7
8	;	;	;r   r  c                     U $ r   r    r#  ms     r   <lambda>r9        Qr   c                 B    UR                  UR                  U 5      5      $ r   r  r  r7  s     r   r9  r9        QWWQWWQZ-@r   c                    [         R                  " SSS9n[        R                  " 5       n[        R                  " S5      n[        R
                  R                  SS/5      nU" Xb5      n[        U[        R                  5      (       d   eXel	        [        R                  " XXE5      nUR                  5       n[        R                  " S[        R                  " 5       4S[        R                  " 5       4S[        R                   " 5       4S	[        R                  " 5       4S
[        R"                  " [        R                  " 5       [        R                   " 5       S.5      4S[        R$                  " 5       4S[        R                   " 5       4/5      n	UR'                  U	5      (       d   e[        R(                  R                  5       n
[        U
[        R                  5      (       d   eg )Nr   Tr   r   r   rb   rc   rd   re   rf   r_   )rh   r   r   r   r   r   discoverrR   PartitioningFactorypartitioning_factoryr   rS  rD   rB   rG   rH   rI   rf   r   r  r  )rs   r  r  rI  r   r   rA  r   rS  rV  hive_partitioning_factorys              r   test_partitioning_factoryrC    sp    DA!!#F))(3G33<<gu=MN"#7G*B,B,BCCCC#7 ))6G (ii	
	

				"((*	299288:BIIK@AB	"((*			! O ""?3333 " 3 3 < < >/1G1GHHHHr   infer_dictionaryc                     U $ r   r    r7  s     r   r9  r9    r:  r   c                 B    UR                  UR                  U 5      5      $ r   r<  r7  s     r   r9  r9    r=  r   c                    [         R                  " SSS9n[        R                  " 5       n[        R                  " S5      n[        R
                  R                  SS/US9nU" Xs5      Ul        [        R                  " XXV5      nUR                  5       n	U(       Gap  [        R                  " [        R                  " 5       [        R                  " 5       5      n
U	R                  S5      R                  U
:X  d   eUR!                  5       R#                  5       R%                  5       nUR'                  S5      R)                  S5      n[        R*                  " S/S	-  S
/S	-  -   5      R-                  5       nUR/                  U5      (       d   eUR!                  5       R#                  [        R                  " S5      S:H  S9nUR'                  S5      R)                  S5      nUR1                  SS	5      nUR/                  U5      (       d   eg U	R                  S5      R                  [        R                  " 5       :X  d   eg )Nr   Tr   r   r   rD  r   r2  r#   r3  r  )rh   r   r   r   r   r   r?  rA  r   rS  rD   r   r   rI   rE   r}   r   r   combine_chunksr=  r   r   r  r  r  )rs   rD  r  r  rI  r   r   rA  r   inferred_schemaexpected_typer   r  r  s                 r   $test_partitioning_factory_dictionaryrL    s    DA!!#F))(3G33<<	%+; = =#*+?#OG ))64G oo'Obhhj"))+>$$U+00MAAA ))+::<e$**1-88UGaK5'A+56HHJ}}X&&&&  ))%E1I)Je$**1->>!Q'}}X&&&&$$U+00BIIK???r   c                     U $ r   r    r7  s     r   r9  r9    r:  r   c                 B    UR                  UR                  U 5      5      $ r   r<  r7  s     r   r9  r9    r=  r   c                    [         R                  " 5       n[        R                  " 5       n[        R
                  " S[        R                  " 5       4/5      n[        R                  " [        R                  " [        S5      5      /US9n[        R
                  " S[        R                  " S5      4S[        R                  " 5       4/5      n[        R
                  " S[        R                  " 5       4S[        R                  " 5       4/5      n[        R
                  " [        U5      [        U5      -   5      nS Hz  n	UR                  U	5        UR                  U	S-   5       n
[        R                  R!                  X5       nUR#                  U5        UR%                  5         S S S 5        S S S 5        M|     [         R&                  " S	S
S9n[        R(                  " S	5      n[        R*                  R-                  US9nU " X5      Ul        [        R0                  " X,X=5      nUR3                  5       nUU:X  d   eUR5                  5       R7                  S[        R8                  " S5      R;                  [        R                  " 5       5      0S9nUS   S   R=                  5       S:X  d   e[        R*                  R-                  SS/SS9nU " X5      Ul        [        R0                  " X,X=5      n[        UR5                  5       R?                  5       5      nUS   R@                  RC                  [        R8                  " S5      S:H  [        R8                  " S5      S:H  -  5      (       d   e[        R*                  " USS9nU " UU5      Ul"        [        R0                  " X,X=5      n[        UR5                  5       R?                  5       5      nUS   R@                  RC                  [        R8                  " S5      S:H  [        R8                  " S5      S:H  -  5      (       d   e[        R*                  R-                  USS9nU " X5      Ul        [        R0                  " X,X=5      n[F        RH                  " [        RJ                  SS9   UR3                  5       nS S S 5        [         R&                  " SS
S9n[        R(                  " S5      n[        RL                  R-                  US9nU " X5      Ul        [        R0                  " X,X=5      nUR3                  5       nUU:X  d   eUR5                  5       R7                  S[        R8                  " S5      R;                  [        R                  " 5       5      0S9nUS   S   R=                  5       S:X  d   e[        RL                  R-                  SS9nU " X5      Ul        [        R0                  " X,X=5      n[        UR5                  5       R?                  5       5      nUS   R@                  RC                  [        R8                  " S5      S:H  [        R8                  " S5      S:H  -  5      (       d   e[        RL                  " USS9Ul"        [        R0                  " X,X=5      n[        UR5                  5       R?                  5       5      nUS   R@                  RC                  [        R8                  " S5      S:H  [        R8                  " S5      S:H  -  5      (       d   e[        RL                  R-                  USS9nU " X5      Ul        [        R0                  " X,X=5      n[F        RH                  " [        RJ                  SS9   UR3                  5       nS S S 5        g ! , (       d  f       GN= f! , (       d  f       GMk  = f! , (       d  f       GN= f! , (       d  f       g = f)Nrb   r   rg   r*   r  rI   )z%directory/2021-05-04 00%3A00%3A00/%24z,hive/date=2021-05-04 00%3A00%3A00/string=%24
/0.featherru   Tr   date_intr.   r   逎`r  r  2021-05-04 00%3A00%3A00%24rB   r  +Could not cast segments for partition fieldr  r   )'rh   ri   r   r,  rD   rB   rG   r   r   r4   r  rI   rm   rk   rl   r  new_filerr   closer   r   r   r?  rA  r   rS  r   r   rE   r  as_pyrP   r  r  r   r  r   rq  r  )r  r  rs   r   rB   r   partition_schemastring_partition_schemafull_schemaru   sinkwriterr   r   rA  r   rJ  r  r$  r   s                       r   *test_partitioning_factory_segment_encodingr_    s    !FFYY
+,-FHHbhhuRy)*6:Eyy
",,s#	$x&=>@ ii
"))+	299; 78:))DL40@+AABK	 	)$&&y<'?@D.&""5) / A@ {d;H))+6G33<< = !#*+?#OG ))&FLGoo'Ok)))^^&&BHHV$))"((*50& F !9Q<:---33<<	V = 5#*+?#OG ))&FLGW^^%3356IQ<,,33	&	6	6	(	u	$	&' ' ' ' ++&:L"<?G))&FLGW^^%3356IQ<,,33	&	6	6	(	u	$	&' ' ' ' 33<<& = :#*+?#OG ))&FLG	rJ
L!//+
L
 v6H))&1G..77 8 !#*+?#OG ))&FLGoo'Ok)))^^&&BHHV$))"((*50& F !9Q<:---..77 8 !#*+?#OG ))&FLGW^^%3356IQ<,,33	&	6	6	(	u	$	&' ' ' ' ..&:G))&FLGW^^%3356IQ<,,33	&	6	6	(	u	$	&' ' ' ' ..77& 8 :#*+?#OG ))&FLG	rJ
L!//+
L 
L_ /. A@R
L 
LN
L 
Ls<   $ ^"^&^^*+^<
^^
^'	*
^9<
_
c                     U $ r   r    r7  s     r   r9  r9  Y  r:  r   c                 B    UR                  UR                  U 5      5      $ r   r<  r7  s     r   r9  r9  Y  r=  r   c                    [         R                  " 5       n[        R                  " 5       n[        R
                  " S[        R                  " 5       4/5      n[        R                  " [        R                  " [        S5      5      /US9n[        R
                  " S[        R                  " S5      4S[        R                  " 5       4/5      n[        R
                  " S[        R                  " 5       4S[        R                  " 5       4/5      n[        R
                  " [        U5      [        U5      -   5      n[        R
                  " S[        R                  " S5      4S[        R                  " 5       4/5      n	[        R
                  " S[        R                  " 5       4S[        R                  " 5       4/5      n
S	nUR                  U5        UR                  US
-   5       n[        R                  R!                  X5       nUR#                  U5        UR%                  5         S S S 5        S S S 5        [         R&                  " SSS9n[        R(                  " S5      n[        R*                  R-                  US9nU " UU5      Ul        [        R0                  " X.X?5      nUR3                  5       nUU:X  d   eUR5                  5       R7                  S[        R8                  " S5      R;                  [        R                  " 5       5      0S9nUS   S   R=                  5       S:X  d   e[        R*                  R-                  SS9nU " UU5      Ul        [        R0                  " X.X?5      n[        UR5                  5       R?                  5       5      nUS   R@                  RC                  [        R8                  " S5      S:H  [        R8                  " S5      S:H  -  5      (       d   e[        R*                  " USS9nU " UU5      Ul"        [        R0                  " X.X?5      n[        UR5                  5       R?                  5       5      nUS   R@                  RC                  [        R8                  " S5      S:H  [        R8                  " S5      S:H  -  5      (       d   e[        R*                  R-                  SS9nU " UU5      Ul        [        R0                  " X.X?5      n[        UR5                  5       R?                  5       5      nUS   R@                  RC                  [        R8                  " S5      S:H  [        R8                  " S5      S:H  -  5      (       d   e[        R*                  " U
SS9nU " UU5      Ul"        [        R0                  " X.X?5      n[        UR5                  5       R?                  5       5      nUS   R@                  RC                  [        R8                  " S5      S:H  [        R8                  " S5      S:H  -  5      (       d   e[        R*                  R-                  U	SS9nU " UU5      Ul        [        R0                  " X.X?5      n[F        RH                  " [        RJ                  SS9   UR3                  5       nS S S 5        g ! , (       d  f       GNy= f! , (       d  f       GN= f! , (       d  f       g = f)Nrb   r   rg   ztest'; dater  ztest';[ string'ztest%27%3B%20dateztest%27%3B%5B%20string%27zLhive/test%27%3B%20date=2021-05-04 00%3A00%3A00/test%27%3B%5B%20string%27=%24rP  r   Tr   rQ  r.   r   rR  ri  r  z2021-05-04 00:00:00$r  rS  rT  rU  rV  r  )&rh   ri   r   r,  rD   rB   rG   r   r   r4   r  rI   rm   rk   rl   r  rW  rr   rX  r   r   r  r?  rA  r   rS  r   r   rE   r  rY  rP   r  r  r   r  r   rq  )r  r  rs   r   rB   r   rZ  r[  r\  partition_schema_enstring_partition_schema_enru   r]  r^  r   r   rA  r   rJ  r  r$  r   s                         r   ;test_partitioning_factory_hive_segment_encoding_key_encodedrf  X  s    !FFYY
+,-FHHbhhuRy)*6:Eyy
c*	+.?-MNP ii
	%(9299;'GHJ))DL40@+AABK))
r||C0	1
%ryy{	3	56 "$
ryy{	+
%ryy{	3	5"61I
i 		"	"9|#;	<VV__T*fu%LLN + 
= v6H))&1G..77 8 !#*+?#OG ))&FLGoo'Ok)))^^&&BHH]+00<0& F !9Q<:---..77 8  #*+?#OG ))&FLGW^^%3356IQ<,,33	-	 $9	9	#	$	+	-. . . . &&%9L"<?G))&FLGW^^%3356IQ<,,33	-	 $9	9	#	$	+	-. . . . ..77 8 !#*+?#OG ))&FLGW^^%3356IQ<,,33	%	&*C	C	-	.%	7	9: : : : &&"V=L"<?G))&FLGW^^%3356IQ<,,33	%	&*C	C	-	.%	7	9: : : : ..77"V 8 =#*+?#OG ))&FLG	rJ
L!//+
L 
Lu +* 
=	<v
L 
Ls0   & Z,"Z(Z, Z>
Z)	$Z,,
Z;>
[c           
         [         R                  " / SQ/ SQS.5      n[        R                  " [         R                  " [         R
                  " S[         R                  " 5       5      [         R
                  " S[         R                  " 5       5      /5      5      n[        R                  " [         R                  5         [        R                  " XSUS9  S S S 5        g ! , (       d  f       g = f)Nr#  yNr#  ri  r  r_   r`   ra   r  r  )rD   r   r   r   rB   rE   rI   r  r   rq  r  r  r   r   s      r   /test_dictionary_partitioning_outer_nulls_raisesrl    s    HH+/BCE??
		288C-rxxRYY[/IJKMD	r	'
DI 
(	'	's   >C
C,c                     [         R                  " / SQ/ SQS.5      n[        R                  " [        5         [
        R                  " XS5        S S S 5        g ! , (       d  f       g = f)Nrh  rj  r_   zbasename-{i}.arrow)rD   r   r  r   r!  r   r  )r  r   s     r   test_positional_keywords_raisesrn    s?    HH+/BCE	y	!
)=> 
"	!	!s   A
A'c                 (   Sn[         R                  " [         R                  " SUS-   5      [        R                  " US-   5      S.5      n[
        R                  " US U U S-  S/S9  [
        R                  " US US-    U S-  S/S9  [
        R                  " U S-  S/S	9nUS   R                  S:X  d   e[
        R                  " U S-  SS
/S	9nUS   R                  S:X  d   e[
        R                  " U S-  S/S	9nUS   R                  S:X  d   eg )Ni   r   r"   )r   r,   r\  r   r  twor.   r,   r{   )	rD   r   repeatnparangerq   r  
read_table
num_chunks)r  
BATCH_SIZEr   s      r   test_read_partition_keys_onlyrw    s    J HHyyJN+:>*, -E kz%1 ozA~%1 MM'E/E7;E<""a'''MM'E/E73CDE<""a'''MM'E/E7;E<""a'''r   c                     [         R                  " U 5      n[        U Vs/ s H?  n[         R                  R	                  [         R                  R                  X5      5      PMA     sn5      $ s  snf r   )oslistdiranyrv   isdirr  )basedirelementsels      r   _has_subdirsr    sG    zz'"H8L8Rbggll7788LMMLs   AA-c                 R   [         R                  " U 5       H  n[         R                  R                  X5      n[         R                  R	                  U5      (       d  MH  [
        R                  " X5      n[        U5      (       a  [        XEU5        M|  UR                  U5        M     g r   )	ry  rz  rv   r  r|  	posixpathr  _do_list_all_dirsr5   )r}  path_so_farrI  r  true_nestednorm_nesteds         r   r  r    si    ZZ ggll7.77==%%#..8KK((!+FCk* !r   c                 $    / n[        U SU5        U$ )Nr  )r  )r}  rI  s     r   _list_all_dirsr    s    Fgr6*Mr   c                 N    [        [        U 5      5      nU[        U5      :X  d   eg r   )r   r  )r  expected_directoriesactual_directoriess      r   _check_dataset_directoriesr    s&    ^G45%9!::::r   c           
      ~   [         R                  " / SQ/ SQS.5      n[        R                  " [         R                  " [         R
                  " S[         R                  " 5       5      [         R
                  " S[         R                  " 5       5      /5      5      n[        R                  " XSUS9  [        U / SQ5        g )	Nrj  rh  r_   r`   ra   r  r  )zx/xzy/yr  )	rD   r   r   r   rB   rE   rI   r  r  rk  s      r   (test_dictionary_partitioning_inner_nullsr    su    HH?1ABCE??
		288C-rxxRYY[/IJKMDUEEw(;<r   c           
         [         R                  " / SQ/ SQS.5      n[        R                  " [         R                  " [         R
                  " S[         R                  " 5       5      [         R
                  " S[         R                  " 5       5      /5      S S5      n[        R                  " XSUS9  [        U / S	Q5        g )
N)r#  Nr  rh  r_   r`   ra   r  r  r  )za=x/b=xz	a=xyz/b=yz	a=z/b=xyz)	rD   r   r   r  rB   rE   rI   r  r  rk  s      r   test_hive_partitioning_nullsr    s    HH+2BCDEryy	#ryy{	#RXXc299;%?@ BCGPDUEEw(MNr   c                     [         R                  " S[         R                  " 5       4S[         R                  " 5       4/5      n SS/n[        R
                  " U 5      n[        U[        R                  5      (       d   e[        R
                  " U SS9n[        U[        R                  5      (       d   e[        R
                  " US9n[        U[        R                  5      (       d   e[        R                  " [        5         [        R
                  " 5         S S S 5        [        R                  " [        SS9   [        R
                  " U S9  S S S 5        [        R                  " [        SS9   [        R
                  " X S9  S S S 5        [        R
                  " U S	S
9n[        U[        R                  5      (       d   e[        R
                  " U SS	S9n[        U[        R                  5      (       d   e[        R
                  " S	S
9n[        U[        R                  5      (       d   e[        R                  " [        5         [        R
                  " US	S
9  S S S 5        [        R                  " [        SS9   [        R
                  " US	S9  S S S 5        [        R                  " [        5         [        R
                  " U SS
9  S S S 5        g ! , (       d  f       GN= f! , (       d  f       GN= f! , (       d  f       GN}= f! , (       d  f       N= f! , (       d  f       N= f! , (       d  f       g = f)Nr   r   inferr  )field_nameszExpected listr  zCannot specify bothr   r  )r  r  zCannot specify 'field_names')r  r  unsupported)rD   rB   r   r   r   r   rR   r   r@  r  r   r^  r  )rB   r  r   s      r   test_partitioning_functionr  	  s   YY,w	.BCDFWE ??6"DdB445555??68DdB223333??u-DdB223333	z	"
 
#	z	9
F+ 
:	z)>	?
3 
@ ??6&1DdB//0000??6GDdB223333??&)DdB223333	z	"
f- 
#	z)G	H
E&9 
I 
z	"
}5 
#	"+ 
#	"	9	9	?	? 
#	"	H	H 
#	"sH   K:K)0K;	L L8L/
K&)
K8;
L

L
L,/
L=c                    [         R                  " [         R                  " S[         R                  " [         R                  " 5       [         R
                  " 5       5      5      [         R                  " S[         R                  " [         R                  " 5       [         R                  " 5       5      5      /5      n[        R                  R                  US9n[        R                  " SSXS9nUR                  R                  U:X  d   eUR                  5       nUR                  S5      R                  R                  UR                   S   5      (       d   eUR                  S5      R#                  5       S/S	-  S
/S	-  -   :X  d   eUR                  S5      R                  R                  UR                   S   5      (       d   eUR                  S5      R#                  5       S/S	-  S/S	-  -   :X  d   eg )Nr   r   rg   r   r   r   r
  r   r   r"   r#   r{   r2  r3  )rD   rB   rE   r   r   r   rI   r   r   r?  rX   r   r   r=  r}   r  typesrx  )rs   rB   r   rX   r   s        r   *test_directory_partitioning_dictionary_keyr  (	  sy    YY
"--	288:>?
bggi=> F ##,,F,;DjjvG &&&000E<< %%,,V\\!_====<< **,a1#'0AAAA<<##**6<<?;;;;<<((*ugkUGaK.GGGGr   c                    [         R                  " [         R                  " S[         R                  " [         R                  " 5       [         R
                  " 5       5      5      [         R                  " S[         R                  " [         R                  " 5       [         R
                  " 5       5      5      /5      n[        R                  R                  US9n[        R                  " SSXS9nUR                  R                  U:X  d   eUR                  5       n[        [        SS5      5      n[        [        S	S
5      5      nUR                  S5      R                  R!                  UR"                  S   5      (       d   eUR                  S5      R$                   H4  nUR                  R'                  5       nUR)                  5         X:X  a  M4   e   UR                  S5      R                  R!                  UR"                  S	   5      (       d   eUR                  S5      R$                   H4  nUR                  R'                  5       nUR)                  5         X:X  a  M4   e   g )Nr   r   rg   r   r   r  i  i  r"      r   )rD   rB   rE   r   r   r   r   r  r?  rX   r   r   rm   r4   r=  r}   r  r  chunksrx  sort)	r   rB   r   rX   r   year_dictionarymonth_dictionaryr   r  s	            r   %test_hive_partitioning_dictionary_keyr  =	  s   YY
rwwy"((*=>
"--	288:>? F ''v'6Djjy]G &&&000E5t,-OE!RL)<<$$++FLLO<<<<f%,,!!++-((( - << %%,,V\\!_====g&--!!++-))) .r   c                     Uc-  [         R                  " [        S5      S/S-  S/S-  -   S.5      nU S-  n[        R                  " XUS9  X4$ )	N	   r7  r   r1  r#   r_   r  r  rD   r   r4   rq   rr   )base_dirr   r  rv   s       r   r3  r3  Y	  sO    }uQxrdQh".ABCn$DNN5~>;r   c                 0   [         R                  " [        S5      S/S-  S/S-  -   S.5      nU S-  n[        R                  " X5        [         R                  " [        SS5      S/S-  S/S-  -   S.5      nU S	-  n[        R                  " X45        X4X$44$ )
Nr  r7  r   r1  r#   r_   ztest1.parquetr  ztest2.parquetr  )r  r   r   r   r   s        r   _create_directory_of_filesr  a	  s    XXE!HB4!8rdQh+>?@F&ENN6!XXE!RLtax2$(/BCDF&ENN6!e^++r   c                     XR                  UR                  U 5      5      4 HV  nU R                  R                  UR                  5      (       d   eUR	                  U 5      R                  U5      (       a  MV   e   g r   )r  r  rB   r  r   )rX   r   r  picklerr   s        r   _check_datasetr  k	  sb    }}W]]7%;<=~~$$U\\2222&&w/66u==== >r   c                 
   [        U [        R                  5      (       d   eU [        U 5      U /[        U 5      /4 HG  n[        R
                  " U 40 UD6n[        U[        R                  5      (       d   e[        XaX#5        MI     [        U R                  5         [        R
                  " U R                  40 UD6n[        U[        R                  5      (       d   e[        XaX#5        S S S 5        g ! , (       d  f       g = fr   )rR   pathlibPathrd   r   rX   r  r  r
   parentname)rv   r   r  r  r   r   rX   s          r   _check_dataset_from_pathr  r	  s    dGLL)))) CIvD	{3**T,V,'2#7#78888w~? 4 
DKK	 **TYY1&1'2#7#78888w~? 
!	 	 s   AC44
Dc                 6    [        U 5      u  p4[        XCX5        g r   r3  r  r  r  r  r   rv   s        r   test_open_dataset_single_filer  	  s    %g.KET.Hr   c                 4    [        U SS9u  p4[        XCX5        g )Nr"   r  r  r  s        r   test_deterministic_row_orderr  	  s    
 &ga@KET.Hr   c                 b    [        U 5      u  p4[        R                  " U5      n[        XX5        g r   )r  rD   concat_tablesr  )r  r  r  tablesr  r   s         r   test_open_dataset_directoryr  	  s(    *73IFV$EW^Kr   c           	         [        U 5      u  nu  pE[        R                  " U5      n[        R                  " XE/5      [        R                  " [        U5      [        U5      /5      /nUU Vs/ s H"  oR                  UR                  U5      5      PM$     sn-  nU HX  n	U	R                  R                  UR                  5      (       d   eUR                  U	5      n
U
R                  U5      (       a  MX   e   g s  snf r   )r  rD   r  r   rX   rd   r  r  rB   r  r   )r  r  r  r  r   r   r   datasetsr   rX   rI  s              r   test_open_dataset_list_of_filesr  	  s    7@FNUV$E 	

E>"


CJE
+,H =E=EM//23X H ~~$$U\\2222((1}}U#### 	s   -)C:c                    [        U 5      u  p[        U5      n[        R                  " U5      nUR                  R                  UR                  5      (       d   e[        R                  " U[        R                  " 5       S9nUR                  R                  UR                  5      (       d   e[        R                  " [        5         [        R                  " U[        R                  " 5       S9  S S S 5        g ! , (       d  f       g = f)Nr  )r3  r   r   rX   rB   r  rh   r   r  r   r!  ri   )r  r   rv   fspathdataset1dataset2s         r   #test_open_dataset_filesystem_fspathr  	  s     &g.KET"F zz&!H??!!%,,//// zz&R-?-?-ABH??!!%,,//// 
y	!


6b&8&8&:; 
"	!	!s   	)C;;
D	c                    U S-  nUR                  5         [        U5      u  pEUR                  U5      n[        R                  " U5      n[        R                  " U[
        R                  " 5       S9n[        R                  " [        U5      [        U5      S9n	UR                  UR                  U5      5      n
UR                  U5      UR                  U5      s=:X  a,  UR                  U	5      s=:X  a  UR                  U
5      :X  d   e   eg )Nsingle-filer  )mkdirr3  relative_tor   rX   rh   r   rd   r	   r  r  r   )r  r  r  ru   r   rv   relative_pathd1d2d3d4s              r   test_construct_from_single_filer  	  s    -'IOO%i0KE$$Y/M 
D	B	DR%7%7%9	:B	C&?93M	NB			]004	5B""2&.*A*A
+ J&&r*J.<.E.Eb.IJ J J J Jr   c                     U S-  nUR                  5         [        U5      u  pE[        R                  " U5      n[        R                  " U[        R
                  " 5       S9n[        R                  " UR                  [        U 5      S9nUR                  U5      n	UR                  U5      n
UR                  U5      nXs=:X  a  U:X  d   e   eXgU4 H:  nUR                  UR                  U5      5      nUR                  U5      U	:X  a  M:   e   g )Nsingle-directoryr  )r  r  r   rX   rh   r   r  r	   r   r  r  )r  r  r  ru   r  r   r  r  r  t1t2t3r   restoreds                 r   $test_construct_from_single_directoryr  	  s    ,,IOO.y9MF	I	B	I"*<*<*>	?B	INNw/G	HB		 	 	$B		 	 	$B		 	 	$B>r>>> b\ &&}':':1'=>&&x0B666 r   c                    U S-  nUR                  5         [        U5      u  p4U Vs/ s H  oUR                  U 5      PM     nn[        U 5         [        R
                  " U5      nUR                  U5      n[        U5      [        [        [        U5      5      :X  d   e S S S 5        [        R
                  " U[        U 5      S9n	UR                  U	5      n
[        R
                  " U5      nUR                  U5      n[        R
                  " U[        R                  " 5       S9nUR                  U5      nWU
s=:X  a  Us=:X  a  U:X  d   e   eg s  snf ! , (       d  f       N= f)Nzlist-of-filesr  )r  r  r  r
   r   rX   r   r   sumrn   r	   rh   r   )r  r  ru   r  r   r   relative_pathsr  r  r  r  r  r  r  t4s                  r   !test_construct_from_list_of_filesr  	  s     /)IOO.y9MF6;<emmG,eN<	G	ZZ'$$R(2w#c#v.//// 

 
Nw/G	HB		 	 	$B	E	B		 	 	$B	Eb&8&8&:	;B		 	 	$BrR =		s   EAE
E#c                     SS/n[         R                  " [        SS9   [        R                  " XS9  S S S 5        g ! , (       d  f       g = f)Nr  z!subdir/1/xxx/doesnt-exist.parquetzdoesnt-existr  r  )r  r   r-  r   rX   )rs   r  s     r   -test_construct_from_list_of_mixed_paths_failsr  	  s;     	%+E 
(	?


5, 
@	?	?s	   <
A
c                    [         R                  " SS/U S9n[         R                  " SU S9n[         R                  " X/5      n[        U[         R                  5      (       d   e[	        [        UR                  5       5      5      S:X  d   eUR                  5       n[	        U5      S:X  d   eUR                  S:X  d   e[	        UR                  5      S:X  d   eUR                   H  nUR                  SS/:X  a  M   e   g )	Nr  r  r  r   r   r  r#   r{   )r   rX   rR   UnionDatasetr   rm   rP   r   rf  childrenr  )rs   r`   ra   rX   r   childs         r   (test_construct_from_mixed_child_datasetsr  

  s     	

002>D	FA


8/Ajj! Ggr////tG))+,-222Eu:!!!w A%%%!!{{;;= = 	= = "r   c                      [         R                  " / SS9n U R                  5       nUR                  S:X  d   eUR                  S:X  d   eg )Nr  r  r   )r   rX   r   r   rf  )emptyr   s     r   test_construct_empty_datasetr   
  sD    JJr%(ENNE>>Q!!!r   c            
      6   [         R                  " / S[        R                  " S[        R                  " 5       4S[        R
                  " 5       4/5      S9n [        R                  " [        SS9   U R                  5         S S S 5        g ! , (       d  f       g = f)Nr  r`   r   rB   zMultiple matches for .*a.* in r  )
r   rX   rD   rB   rG   rI   r  r   r^  r   )r  s    r   *test_construct_dataset_with_invalid_schemar  '
  sh    JJr%			bhhj	biik; 1 E 
z)I	J 
K	J	Js   0B


Bc                   ^ [         R                  " U [        R                  " S5      [         R                  " 5       S9n[         R                  " U [        R                  " S5      [         R                  " 5       S9n[
        R                  R                  [
        R                  " [        S5      5      /S/S9m[
        R                  R                  [
        R                  " [        S5      5      /S/S9n[        R                  " [        SS	9   [         R                  " X/5        S S S 5        S
n[        R                  " [        US	9   [         R                  " / SQ5        S S S 5        Sn[        R                  " [        US	9   [         R                  " S 5        S S S 5        Sn[        R                  " [        US	9   [         R                  " U4S j[        S5       5       5        S S S 5        Sn[        R                  " [        US	9   [         R                  " / 5        S S S 5        Sn[        R                  " [        US	9   [         R                  " TU/5        S S S 5        Sn[        R                  " [        US	9   [         R                  " TS/5        S S S 5        Sn[        R                  " [        US	9   [         R                  " TS/5        S S S 5        g ! , (       d  f       GN= f! , (       d  f       GN= f! , (       d  f       GNo= f! , (       d  f       GN4= f! , (       d  f       GN= f! , (       d  f       N= f! , (       d  f       N= f! , (       d  f       g = f)Nr\  r  /schemar   r`   r  ra   z"Expected.*FileSystemDatasetFactoryr  zExpected a list of path-like or dataset objects, or a list of batches or tables. The given list contains the following types: intr  zbExpected a path-like, list of path-likes or a list of Datasets instead of the given type: NoneTypezcExpected a path-like, list of path-likes or a list of Datasets instead of the given type: generatorc              3   (   >#    U  H  nTv   M	     g 7fr   r    )r  r  batch1s     r   r  <test_construct_from_invalid_sources_raise.<locals>.<genexpr>U
  s     -HqFH   r^   zEMust provide schema to construct in-memory dataset from an empty listzFItem has schema
b: int64
which does not match expected schema
a: int64z}Expected a list of path-like or dataset objects, or a list of batches or tables. The given list contains the following types:r   zCExpected a list of tables or batches. The given list contains a int)r   r   rh   r   r   rD   r<  r   r   r4   r  r   r!  rX   r^  InMemoryDataset)r   child1child2batch2r  r  s        @r   )test_construct_from_invalid_sources_raiser  0
  so   ((
!##%F
 ((
	"##%F
 ^^''%))<(=cU'KF^^''%))<(=cU'KF	y(L	M


F#$ 
N	 
 
y	1


9 
2	.  
y	1


4 
2	/  
y	1


-E!H-. 
2 	P  
z	2
2 
3	  
y	1


FF#$ 
2	J  
y	1


FA; 
2 	N  
y	1
FA;' 
2	1a 
N	M 
2	1 
2	1 
2	1 
3	2 
2	1 
2	1 
2	1s`   K!
K3L *LL)L;M?M!
K03
L
L
L&)
L8;
M	
M
M+c                 V   [         R                  R                  [         R                  " [	        S5      5      /S/S9n[         R
                  R                  UR                  U/5      n[         R                  R                  U/5      n[        R                  " / S[         R                  " / 5      S9R                  5       nU[         R                  " / 5      :X  d   eXU/U/U4 H  n[        R                  " U5      nU R                  U5      U:X  d   e[        [        UR                  5       5      5      S:X  d   e[!        UR                  5       5      R                  5       U:X  d   e[         R                  R                  [        UR#                  5       5      5      U:X  a  M   e   g )Nr   r`   r  r  r  r"   )rD   r<  r   r   r4   RecordBatchReaderrp   rB   rJ   r   rX   r   r   r   rm   rP   r7   r   )r  ry   r  r   dataset_tablesourcerX   s          r   test_construct_in_memoryr  r
  sL   NN&&r(;'<SE&JE!!..u||eWEFHH!!5'*EJJr%		" !!) BHHRL(((%5'6:**V$&&w/58884--/01Q666G))+,5575@@@xx$$T'*<*<*>%?@EIII ;r   r   c                 (  ^^ [         R                  R                  [         R                  " [	        S5      5      /S/S9m[         R
                  R                  T/5      mSnU4S jS 4U4S jS 4U4S jTR                  44 Hz  u  p#[        R                  R                  U" 5       X0S9nUR                  5       T:X  d   e[        R                  " [         R                  US	9   UR                  5         S S S 5        M|     g ! , (       d  f       M  = f)
Nr   r`   r  z#OneShotFragment was already scannedc                  Z   > [         R                  R                  T R                  T /5      $ r   )rD   r  rp   rB   ry   s   r   r9  $test_scan_iterator.<locals>.<lambda>
  s!    R))66ug'r   c                     > [        T 5      $ r   )r   r   s   r   r9  r  
  s    '.r   c                  .   > U 4S j[        S5       5       $ )Nc              3   (   >#    U  H  nTv   M	     g 7fr   r    )r  r  ry   s     r   r  7test_scan_iterator.<locals>.<lambda>.<locals>.<genexpr>
  s     .XeXr  r"   )r4   r  s   r   r9  r  
  s    .U1X.r   rB   r   r  )rD   r<  r   r   r4   rJ   rp   rB   r   rp  r   r  r   rq  )r   r  r   rB   r   ry   r   s        @@r   test_scan_iteratorr  
  s    NN&&r(;'<SE&JEHH!!5'*E1E'(,..5.=	 **))If * ?!U***]]2??%8 98 98s   $D
D	c                    [         R                  " [        S5      S/S-  S/S-  -   S.5      nU S-  nUR                  5         [        S5       HH  nUS	U 3-  nUR                  5         [        R
                  " UR                  SU-  S5      US
-  5        MJ     UR                  S[         R                  " S/S-  S/S-  -   S/S-  -   [         R                  " 5       S95      nXR4$ )Nr  r7  r   r1  r#   r_   zdataset-partitionedr^   zpart=r  r   r   r"   r{   r|   )
rD   r   r4   r  rq   rr   r  append_columnr   r   )r}  r   rv   r?   r   
full_tables         r   _create_partitioned_datasetr  
  s    HH582$(bTAX*=>?E**DJJL1XaSk!


u{{1Q3*D>,AB 
 $$!qA37*aS1W4288:FHJ r   c           
          [        U 5      u  p4UR                  SS/5      n[        XEX5        [        R                  " [        U5      [        R                  " SS9S9nUR                  R                  UR                  5      (       d   e[        U 5         [        R                  " S[        R                  " SS9S9nUR                  R                  UR                  5      (       d   e S S S 5        [        R                  " [        U5      SS9nUR                  R                  UR                  5      (       d   e[        R                  " [        U5      [        R                  " [        R                  " S[        R                  " 5       4/5      SS9S9nUR                  R                  [        R                  " S[        R                  " 5       5      5      nUR                  R                  U5      (       d   eUR                  5       nUR                  S[        R                   " S/S	-  S
/S	-  -   S/S	-  -   [        R                  " 5       S95      n	UR                  U	5      (       d   eg ! , (       d  f       GN= f)Nr`   ra   r   r  r   zdataset-partitioned/r   r   r^   r"   r{   r|   )r  r  r  r   rX   rd   r   rB   r  r
   rD   r   r5   rE   r   r   r   )
r  r  r  r  rv   r   rX   rV  rI  r  s
             r   'test_open_dataset_partitioned_directoryr  
  s   27;J sCj)ET.H jjD	v >@G>>  !2!23333 
G	**3*,//*HJ~~$$Z%6%67777 
 jjT8G>>  !2!23333 jjD	__II	*+,V=>G ll))"((62779*EFO>>  1111F""!qA37*aS1W42779EGH=="""") 
	s   AI..
I=c                 *   [        U 5      u  p[        R                  " [        U5      5      nUR                  R                  UR                  5      (       d   e[        R                  " [        U5      [        R                  " 5       S9nUR                  R                  UR                  5      (       d   e[        U 5         [        R                  " S[        R                  " 5       S9nS S S 5        WR                  R                  UR                  5      (       d   e[        R                  " [        5         [        R                  " [        U5      [        R                  " 5       S9  S S S 5        g ! , (       d  f       N= f! , (       d  f       g = f)Nr  r  )r3  r   rX   rd   rB   r  rh   r   r
   r  r   r-  ri   )r  r   rv   r  r  dataset3s         r   test_open_dataset_filesystemr  
  s    &g.KE zz#d)$H??!!%,,//// zz#d)0B0B0DEH??!!%,,//// 
G	::n9K9K9MN 
??!!%,,//// 
(	)


3t9););)=> 
*	) 
	
 
*	)s   )E382F3
F
Fc                     [        U 5      u  p[        R                  " [        SS9   [        R
                  " U/SS9  S S S 5        g ! , (       d  f       g = f)Nz format 'blabla' is not supportedr  blablar  )r3  r  r   r^  r   rX   )r  r  rv   s      r   $test_open_dataset_unsupported_formatr  
  s:    !'*GA	z)K	L


D6(+ 
M	L	Ls   A
Ac                 F   [        U 5      u  p4[        R                  " U5      n[        R                  " XU/5      n[        U[        R                  5      (       d   eUR                  UR                  U5      5      nUR                  U5      UR                  U5      :X  d   eg r   )r3  r   rX   rR   r  r  r  r   )r  r  r  r  rv   rX   unionr  s           r   test_open_union_datasetr  
  s    !'*GAjjGJJ)*EeR__----!!-"5"5e"<=G""7+~/F/Fu/MMMMr   c                     [         R                  " SU SS9n[        R                  " [        SS9   [         R                  " U/SS9  S S S 5        g ! , (       d  f       g = f)Nr\  r   r]  zcannot pass any additionalr  r  )r   rX   r  r   r^  )r   r  s     r   .test_open_union_dataset_with_additional_kwargsr  
  s@    JJxM)LE	z)E	F


E79- 
G	F	Fs   A
Ac                  :   [         R                  " [        5         [        R                  " SSS9  S S S 5        [         R                  " [
        R                  SS9   [        R                  " SSS9  S S S 5        g ! , (       d  f       NP= f! , (       d  f       g = f)Nzi-am-not-existing.arrowr  r  zcannot be relativer  zfile:i-am-not-existing.arrow)r  r   r-  r   rX   rD   rq  r    r   r   #test_open_dataset_non_existing_filer  
  se     
(	)


,U; 
* 
r.B	C


1%@ 
D	C 
*	) 
D	Cs   A;B;
B	
Br   ru   r   r  r  partition_keysr  BCr  )DEFr  )r"   Nr^   )r  Nr  )Nr{   r^   c           	        ^ [         R                  " [        S5      S/S-  S/S-  -   S.5      nS US   ;   =(       d    S US   ;   nUS	:X  a  U(       a  g US	:X  a%  [        R                  R                  S
S/TS9nSnS nOQU(       a  [        R                  R                  TUS9nO[        R                  R                  TS9nSnU(       a  UnOSnU S-  n	U	R                  5         Uu  pU
 H[  nU HR  nU	UR                  U=(       d    UU=(       d    U5      -  nUR                  SS9  [        R                  " X^S-  5        MT     M]     [        R                  " [        U	5      US9nU4S jnUR                  R                  [         R                  " S
U" U
S   5      5      5      R                  [         R                  " SU" US   5      5      5      nUR                  R!                  U5      (       d   eg )Nr  r7  r   r1  r#   r_   r   r"   ru   part1part2rH  z{0}/{1})rD  r  zpart1={0}/part2={1}__HIVE_DEFAULT_PARTITION__rX   T)parentsr  r  c                 b  > T(       ai  [        U [        5      (       a  [        R                  " 5       O[        R                  " 5       n[        R
                  " [        R                  " 5       U5      $ [        U [        5      (       a  [        R                  " 5       $ [        R                  " 5       $ r   )rR   rd   rD   rI   r   r   )r   
value_typerD  s     r   rK  /test_partition_discovery.<locals>.expected_type?  s_    (23(<(<"((*J==Z88",S#"6"6299;FBHHJFr   )rD   r   r4   r   r   r?  r  r  r   rq   rr   rX   rd   rB   r5   rE   r  )r  r   r  rD  r  r   has_nullfmt
null_valuebasepath
part_keys1
part_keys2r  r  rv   rX   rK  rV  s      `              r   test_partition_discoveryr(    s   " HH583%!)seai*?@AE~a((EDN14E,EH{"x{"//88g1A 9 C
..77!1 8 L ..77!1 8 3L#&J5J"HNN+JE

5.J0CDEDJJtJ$NN5"78	    jjX\BGG ll))
-
167f
-
167 
 >>  1111r   c                    [         R                  " [        R                  " SS/S5      [	        S5      S.5      n[
        R                  " UR                  S/5      R                  SS9n[
        R                  " X US	S
9  [
        R                  " U S	[
        R                  R                  SS9S9n[         R                  " US   US   R                  5       S.5      nUR                  5       R                  U5      (       d   e[!        UR#                  5       5      S   nUR                  UR                  S9R                  US S 5      (       d   eUR$                  nUR'                  UR)                  U5      5      nUR                  5       R                  U5      (       d   eUR'                  UR)                  U5      5      nUR                  UR                  S9R                  US S 5      (       d   eUR                  UR                  S9R+                  5       R                  US S R+                  5       5      (       d   eUR$                  R                  U5      (       d   eg )Nr  r  r#   r   r   r  r   r   r  r1  r   r   TrH  r  r  )r  r   r   rg   )rD   r   rr  rq  r4   r   r   r  rB   r  rX   r  r?  r  r   r  rm   rP   r  r  r  r  )	r  r  r   r   rX   r  rY   	part_exprr  s	            r   4test_dataset_partitioned_dictionary_type_reconstructr-  M  s    HHbiic
A6uRyIJE??5<<188HDU$yIjj	((1141HG xxeeFm&E&E&GHH $$X....G))+,Q/HGNN3::8BQ<HHHH--I""=#6#6w#?@H%%h////""=#6#6x#@AHGNN3::8BQ<HHHHGNN3==?FF!     ((//	::::r   c           	      J   SSK Jn  U S   u  p#pESU SU SU SU S3	nUR                  U5      u  pxUR                  " S5        [        R
                  " S	/ S
Q05      n	UR                  " S5       n
[        R                  " X5        S S S 5        XXvX#XE4$ ! , (       d  f       N= f)Nr   
FileSystem
connections3://:z5@mybucket/data.parquet?scheme=http&endpoint_override=z&allow_bucket_creation=Truemybucketr`   r  zmybucket/data.parquet)	r   r0  from_urirk   rD   r   rl   rq   rr   )	s3_serverr0  rj  rk  rl  rm  ri  rh   rv   r   rw   s              r   rh  rh  l  s    %)2<)@&D


|1ZL )"V1TF*E	G 
 ""3'HBMM*HHc9%&E			6	73
u" 
8 ZCC 
8	7s   /B
B"c                    U u  p#pE      n[         R                  " USS9nUR                  U5      R                  U5      (       d   e[         R                  " USUS9nUR                  U5      R                  U5      (       d   eg )Nr   r  r   r
  )r   rX   r   r  )rh  r  r   rv   rh   ri  r  rX   s           r   test_open_dataset_from_uri_s3r9    s     (9$E!Q1 jjY/G""7+2259999 jjiB?G""7+2259999r   c                     U u  p#pE      n[         R                  " S5      nUR                  U5      n[        R                  " USUS9n	UR                  U	5      R                  U5      (       d   eg )Nr4  r   r8  )rh   r   rd  r   rX   r   r  )
rh  r  r   rv   r
  ri  r  r   finfosrX   s
             r    test_open_dataset_from_fileinfosr<    si     0A,E!Q1z*H%%h/Fjj	jIG""7+2259999r   c           	         U u  p  p4pVn[         R                  " S5      nSSKJn	Jn
  UR                  UUSSU SU 30S9n[        R                  " USUS	9nUR                  5       R                  U5      (       d   eU
" U	" U5      5      n[        R                  " USUS	9nUR                  5       R                  U5      (       d   eg )
Ns3fsr   )FSSpecHandlerr   endpoint_urlzhttp://r3  )r   secretclient_kwargsr   r8  )
r  importorskipr   r?  r   S3FileSystemr   rX   r   r  )rh  r   rv   r  rj  rk  rl  rm  r>  r?  r   rh   rX   s                r   $test_open_dataset_from_uri_s3_fsspecrE    s     =N9EATzv&D6			gdV1TF3
 
 
B jjiB?G$$U++++ 
mB'	(BjjiB?G$$U++++r   c                    SSK Jn  U S   u  p#pESnSnSU SU SU S	U S
U SU S3nUR                  U5      u  pUS:X  d   eU	R                  " U5        [        R
                  " S/ SQ05      n
U	R                  " U5       n[        R                  " X5        S S S 5        [        R                  " USS9nUR                  5       R                  U
5      (       d   eSR                  XEX#5      n/ SQnU HQ  u  pUR                  U5      n[        R                  " XxSS9nUR                  5       R                  U
5      (       a  MQ   e   [        R                  " [        R                   SS9   UR                  S	5      n[        R                  " SUS9  S S S 5        SnSnUR                  U5      n[        R                  " ["        5       n[        R                  " SUS9  S S S 5        [%        WR&                  5      UR                  SXx5      :X  d   eSnUR                  U5      n[        R                  " ["        5       n[        R                  " SUS9  S S S 5        [%        UR&                  5      UR                  SXx5      :X  d   eg ! , (       d  f       GN= f! , (       d  f       GN= f! , (       d  f       N= f! , (       d  f       No= f)Nr   r/  r1  theirbucketnested/folder/data.parquetr2  r3  @r   z?scheme=http&endpoint_override=z&allow_bucket_creation=truez&theirbucket/nested/folder/data.parquetr`   r  r   r  3s3://{}:{}@{{}}?scheme=http&endpoint_override={}:{}))ztheirbucket/nested/folder/z/data.parquet)ztheirbucket/nested/folderdata.parquet)ztheirbucket/nested/folder/data.parquet)ztheirbucket/nestedrL  )rG  z/nested/folder/data.parquet)rG  rH  r]  zMissing bucket namer  z'/theirbucket/nested/folder/data.parquetr  zThe path component of the filesystem URI must point to a directory but it has a type: `{}`. The path component is `{}` and the given filesystem URI is `{}`ztheirbucket/doesnt/existrK  NotFoundFile)r   r0  r5  rk   rD   r   rl   rq   rr   r   rX   r   r  r   r  r   rq  r^  rd   r,   )r6  r0  rj  rk  rl  rm  bucketrv   ri  rh   r   rw   rX   templater  prefixrC  excs                     r   -test_open_dataset_from_s3_with_filesystem_urirS    sm    &)2<)@&D
F'D

|1ZL&4& 9"V1TF*E	G 
 ""3'HB;;;;MM&HHc9%&E			t	$
u" 
% jjY/G$$U++++ 	>DDD	
 
E oof%**T)D!((//// 
 
r.C	Dooc"


<M 
E
	! 
 &D
//$
C	z	"c


>c2 
#syy>U\\*d@@@@3D
//$
C	z	"c


>c2 
#syy>U\\&$<<<<] 
%	$4 
E	D 
#	" 
#	"s0   J	+'J	J-?J>	
J
J*-
J;>
Kc                     [        U 5      u  p[        R                  " S5      nUR                  S5      n[        R
                  " X$S9nUR                  R                  UR                  5      (       d   eg )Nfsspecfiler  )r3  r  rC  r
  r   rX   rB   r  )r  r   rv   rU  r   rX   s         r   test_open_dataset_from_fsspecrW    s[    %g.KE  *F'Gjj2G>>  ....r   c                 n   [         R                  " S5      n[        R                  " S/ SQ05      nU S-  n[        R
                  " X#5        UR                  S5      nUR                  U 5      S   R                  S5      (       d   e[        R                  " 5       n[        R                  " [        R                  " U5      5      nUR                  X65      nUR                  UR                   5      (       d   eUR#                  X65      nUR$                  R                  UR                   5      (       d   eg )NrU  r`   r  rK  rV  r   )r  rC  rD   r   rq   rr   r
  lsendswithr   r   rh   r   r?  rS  r  rB   r  r  )	r  rU  r   rv   	fsspec_fsr   r
  rB   rY   s	            r   test_file_format_inspect_fsspecr\    s       *F HHc9%&E^#DNN5 !!&)I<< #,,^<<<< !!#F !1!1)!<=J^^D-F==&&&&##D5H##**5<<8888r   c                 "   U S-  n[         R                  " SS/S-  [        S5      S.5      n[        R                  " UR                  S/5      R                  SS	9n[        R                  " X2US
S9  [        R                  " [         R                  " S[         R                  " S5      4/5      SS	9n[        R                  " US
US9n[        R                  " S5      [        R                  " S5      :  nUR                  XVS9nUR                  S5      R                  5       / SQ:X  d   eSS Kn[        R                  " S5      UR                   " SSS5      :  nUR                  XVS9nUR                  S5      R                  5       / SQ:X  d   eg )Ntest_partition_timestamps
2012-01-01z
2012-01-02r#   r   )datesr  r`  r   r  r1  r+  r  r  r  r  )r"   r^   r#   re  r  r   i  r"   )rD   r   r4   r   r   r  rB   r  r  rX   rE   r8   	Timestampr   r=  rx  r0   )r  r  rv   r   r   rX   rH  r0   s           r   test_filter_timestamprb    sS    00DHH-1Bi E
 ??5<<	299&IDUtIF ??299wS0A&B%CD"(*DjjidCG!BLL$>>I##G#>E<<'')_<<<!H$5$5dAq$AAI##G#>E<<'')_<<<r   c                 J   [         R                  " S[         R                  " / SQ[         R                  " 5       S905      n[	        X5      u  p4[
        R                  " [        U5      5      n[
        R                  " S5      S:  n[        UR                  XVS95      S:X  d   eg )Nr`   )r   r"   r{   r^   r   r#   r|   r{   r  r^   )rD   r   r   r   r3  r   rX   rd   rE   r   r   )r  r  r   r  rv   rX   filter_s          r   test_filter_implicit_castre  8  sz     HHc288$6RWWYGHIE!'1GAjjT#GhhsmaG~&&w&?@AEEEr   c                 *   [         R                  " S/ SQ05      n[        X5      u  p4[        R                  " [        U5      5      nUR                  U[        R                  " S5      [        R                  " S 5      :H  S9nUR                  S:X  d   eg )Nr  )r`   ra   Nr  r   )
rD   r   r3  r   rX   rd   r   rE   r  r   )r  r  r   r  rv   rX   s         r   test_filter_equal_nullrg  C  sy     HHc+,-E!'1GAjjT#G##48 $ E >>Qr   c                    [         R                  " / SQ[        S5       Vs/ s H  n[        R                  " SSSU5      PM     sn[        SS5       Vs/ s H  n[        R                  " SSU5      PM     snS.5      n[	        X5      u  pE[
        R                  " [        U5      5      n[        R                  " [
        R                  " S5      [         R                  " SS	/5      5      nUR                  XgS
9R                  S:X  d   e[        R                  " [
        R                  " S5      5      S:  nUR                  XgS
9R                  S:X  d   e[        R                  " [
        R                  " S5      [
        R                  " S5      5      nUR                  USU0S9n	U	S   R!                  5       / SQ:X  d   eg s  snf s  snf )N)r`   ra   Nr`   r   r#   i  r"   r  r  r  r`   ra   r  r^   r  r{   r  r$   r.   r0  )rD   r   r4   r0   r3  r   rX   rd   r  is_inrE   r   r   r   hourdays_betweenrx  )
r  r  r?   r   r  rv   rX   rd  r$   rI  s
             r   test_filter_compute_expressionrl  P  sg   HH'8=aA1haA.A5:1a[A[ha+[A E
 "'1GAjjT#Ghhrxx}bhhSz&:;G""7";DDIIIggbhhsm$)G""7";DDIII??288C="((3-8D$$Wvtn$EF&>##%888 BAs   #G
"G	c                 l   [         R                  " U [        R                  " S5      [         R                  " 5       S9n[         R
                  " U/5      n[        UR                  5       5      S:X  d   e[        S UR                  5        5       5      (       d   eUR                  5       S   R                  UR                  5       5      (       d   eUR                  5       R                  UR                  5       5      (       d   e[        UR                  5       [         R                  5      (       d   eg )Nr\  r  r"   c              3   V   #    U  H  n[        U[        R                  5      v   M!     g 7fr   )rR   rD   r;  )r  r  s     r   r  %test_dataset_union.<locals>.<genexpr>n  s      K1JAz!RYY''1Js   ')r   )r   r   rh   r   r   UnionDatasetFactoryr   rQ  r  r  rS  rR   r   r:  )r   r  r   s      r   test_dataset_unionrq  e  s    ''rx0##%E $$eW-G w&&()Q...K1H1H1JKKKKK""$Q'..u}}??????##EMMO4444gnn&

3333r   c                 	   [         R                  " SUSS9n[         R                  " SUSSS/S9n[         R                  " SUSS	S9nUR                  UR                  s=:w  a  UR                  :w  d   e   e[         R                  " X#U/5      n[        U[         R                  5      (       d   eS
n[
        R                  " [        US9   [         R                  " X#/US9  S S S 5        [        R                  " S[        R                  " 5       4S[        R                  " 5       4S[        R                  " 5       4S[        R                  " 5       4S[        R                  " 5       4S[        R                  " 5       4S[        R                  " 5       4/5      nUR                  R                  U5      (       d   eUR                  5       R                  R                  U5      (       d   e[         R                  " X$/5      n[        R                  " S[        R                  " 5       4S[        R                  " 5       4S[        R                  " 5       4S[        R                  " 5       4S[        R                  " 5       4S[        R                  " 5       4/5      nUR                  R                  U5      (       d   eUR                  5       R                  R                  U5      (       d   e[        R                  " S[        R                  " 5       4S[        R                  " 5       4S[        R                  " 5       4/5      n[         R                  " X$/US9nUR                  5       R                  R                  U5      (       d   e[        R                  " S[        R                  " 5       4S[        R                  " 5       4S[        R                  " 5       4/5      n[         R                  " X$/US9nUR                  5       R                  R                  U5      (       d   e[        R                   " [#        S5      S/S-  S/S-  -   S// SQS9n[%        XS9u  p[         R                  " U
5      n[
        R                  " [        R&                  SS9   [         R                  " X+/5        S S S 5        g ! , (       d  f       GN= f! , (       d  f       g = f)Nr\  r   r]  r  weekr-   r
  r   r   /hiver   z$cannot pass any additional argumentsr  r  r*   r+   r,   r   r   rg   rj  r  r7  r   r1  r#   	abcdefghj)r*   r,   r+   r  r   zUnable to merge)r   rX   rB   rR   r  r  r   r^  rD   rF   rG   rH   rI   r   r  r   r   r4   r3  rR  )r  r   r  r  child3	assembledmsgrV  r   r  rv   child4s               r   &test_union_dataset_from_other_datasetsr{  t  s   ZZ]9MFZZ	mI&,g%68FZZM)%+-F ==FMM:V]]:::::

FF34Ii1111
0C	z	-


F#> 
. ii		"((*	"**,	"))+			"((*! O ""?3333&&--o>>>>

F+,Iii		"((*	"**,	"))+		"((*! O ""?3333&&--o>>>>ii	"((*	"))+	! O
 

F+ODI&&--o>>>>ii	"((*	"))+	BIIK ! O
 

F+ODI&&--o>>>> HHeAhqB4!8 3[A57E!'7GAZZF	r((0A	B


F#$ 
C	Bc 
.	-b 
C	Bs   R>S>
S
Sc                     Sn[         R                  " [        US9   [        R                  " / SQU S9  S S S 5        g ! , (       d  f       g = f)Nz8points to a directory, but only file paths are supportedr  )r\  r  ru  r  )r  r   IsADirectoryErrorr   rX   )r   ry  s     r   4test_dataset_from_a_list_of_local_directories_raisesr~    s1    
DC	(	4


1mL 
5	4	4s	   =
Ac           
         [         R                  " [         R                  " SU S9[         R                  " SU S9[         R                  " SU S9/5      n[        R                  " S[        R                  " 5       4S[        R
                  " 5       4S[        R                  " 5       4S[        R                  " 5       4/5      nUR                  R                  U5      (       d   e[         R                  " [         R                  " SU S9[         R                  " SU S9[         R                  " SU S	S
9/5      n[        R                  " S[        R                  " 5       4S[        R
                  " 5       4S[        R                  " 5       4S[        R                  " 5       4S[        R                  " 5       4S[        R                  " 5       4/5      nUR                  R                  U5      (       d   eg )Nr\  r  r  ru  r*   r+   r,   r-   r   )r
  r   r   r   )
r   rX   rD   rB   rF   rG   rH   rI   r  r   )r   rX   rV  s      r   &test_union_dataset_filesystem_datasetsr    sp   jj


86


97


7}5 G
 ii		"((*	"**,	"))+	! O >>  1111 jj


86


97


7}6J G
 ii		"((*	"**,	"))+		"((*! O >>  1111r   c                   ^ ^ [         R                  " / SQ/ SQS.5      n[        R                  " UT S-  5        SUU 4S jjnS nUnU" XEUR                  S9  UR                  nUnU" XE5        [         R                  " SS/5      n[         R                  " / SQ/ SQ/S	S
/S9nU" XE5        [         R                  " S/5      n[         R                  " / SQ/S
/S9nU" XE5        [         R                  " SS/5      n[         R                  " / SQ[         R
                  " / SQSS9/S
S/S9nU" XE5        [         R                  " SS/5      n[        R                  " [        T S-  5      US9n[         R                  " US
   R                  S5      US	   /S
S	/S9nU" XE5        [         R                  " S
[         R                  " [         R                  " 5       5      4S/5      n[        R                  " [        T S-  5      US9nUR                  R                  U5      (       d   e[        R                  " [        SS9   TR!                  U5        S S S 5        g ! , (       d  f       g = f)Nr  r"  r#  r$  r_   rK  c                 *  > [         R                  " [        TS-  5      U S9nUb#  UR                  R	                  U5      (       d   eO"UR                  R	                  U 5      (       d   eTR                  U5      nUR	                  U5      (       d   eg )NrK  rg   )r   rX   rd   rB   r  r   )rB   r  rV  rX   rI  r  r  s        r   r  -test_specified_schema.<locals>._check_dataset  s~    **S>!9:6J&>>((9999>>((0000((1}}X&&&&r   )rV  )ra   rH   )r`   rG   ra   r`   r  )r   r   NNNr   r|   r   )r`   r   rg   z#Unsupported cast from int64 to listr  r   )rD   r   rq   rr   rB   r   r   rX   rd   r  list_r   r  r  r   r+  r   )r  r  r   r  rB   r  rX   s   ``     r   test_specified_schemar    s   HH9<89ENN5'N23' ' FH6U\\B \\FH6$ YY(.9:Fxxy1#sDH6$ YY'(FxxC51H6$ YY78Fxx"47CE"Cj*H 6$ YY(89:FjjW~56vFGxxs1s%"Cj*H 6$ YYbhhrxxz235EFGFjjW~56vFG>>  ((((	*B
D(
D 
D 
Ds   ?I
I(c                    U S-  n[         R                  " S/ SQ05      n[        R                  " X25        [         R                  " S[         R
                  " 5       4/5      n[        R                  " [        U5      /S-  US9nUR                  R                  U5      (       d   eUR                  U5      n[        R                  " [        SS9   UR                  5       nUR                  5         S S S 5        g ! , (       d  f       g = f)NrK  r`   r  d   rg   z#Unsupported cast from int64 to nullr  )rD   r   rq   rr   rB   r  r   rX   rd   r  r   r  r   r+  rt  ru  )r  r  fnr   rB   rX   r   r  s           r   test_incompatible_schema_hangr    s     
>	!BHHc9%&ENN5YYbggi()*Fjj#b'S8G>>  (((($$W-G	*B
D""$
D 
D 
Ds   !C::
Dc                    [         R                  " [         R                  " / SQSS9[         R                  " / SQSS9S.5      n[        U S-  5      n[         R                  " U5       n[         R
                  " XBR                  5      nUR                  UR                  5       S   5        UR                  5         S S S 5        [        R                  " U[        R                  " 5       S	9nUR                  U5      nUR                  U5      (       d   e[        U5        S
 H@  n[        R                  " X8S	9nUR                  U5      nUR                  U5      (       a  M@   e   g ! , (       d  f       N= f)Nr  r   r|   r  rH   r_   z
test.arrowr   r  )r  arrow)rD   r   r   rd   output_streamRecordBatchFileWriterrB   write_batchr   rX  r   rX   r,  r   r  r[   )	r  r  r   rv   r]  r^  rX   rI  
format_strs	            r   test_ipc_formatr  (  s   HH288IF;88LyAC DE w%&D			$	4))$=5++-a01 
 
 jjb&6&6&89G$$W-F==/8&
**T5((1}}U#### ' 
 	s   (AE&&
E4c           
         SSK Jn  [        R                  " [        R                  " / SQSS9[        R                  " / SQSS9S.5      n[        U S	-  5      nUR                  X45        [        R                  " U[        R                  " 5       S
9n[        UR                  5       5      n[        US   [        R                  5      (       d   eUR                  U5      nUR                  SS9  UR!                  U5      (       d   e[#        U5        [        R                  " USS
9nUR                  U5      nUR                  SS9  UR!                  U5      (       d   eUR                  US/S9nUR                  SS9  UR!                  UR%                  S/5      5      (       d   eUR                  US[        R&                  " S5      S-  0S9nUR                  SS9  UR!                  [        R                  " S[        R                  " / SQSS905      5      (       d   eUR)                  U5      S:X  d   eUR)                  U[        R&                  " S5      S:  S9S:X  d   eg )Nr   orcr  r   r|   r  rH   r_   test.orcr  TrP  r  ra   r.   b2r{   )r#  r%  g333333?r^   r`   r  r"   )r   r  rD   r   r   rd   rr   r   rX   r8  rm   rP   rR   FileFragmentr   rT  r  r[   r  rE   r  )r  r  r  r   rv   rX   r$  rI  s           r   test_orc_formatr  >  s   HH288IF;88LyAC DE w#$DOOE jjb&6&6&89GW**,-IilBOO4444$$W-F
OOO==/8jje,G$$W-F
OOO==$$Wse$<F
OOO==se,----$$$ 12 % F OOO==
$I>?@    $$W-222$$WRXXc]Q5F$G1LLLr   c                 n   SSK Jn  [        R                  " [        R                  " / SQSS9[        R                  " / SQSS9S.5      n[        U S	-  5      nUR                  X45        [        R                  " US
S9n[        UR                  U5      5      n[        U5      S:X  d   eUS   R                  S:X  d   eUS   R                  UR                  5       S   5      (       d   e[        UR                  USS95      n[        U5      S:X  d   eUS   R                  S:X  d   eUS   R                  UR                  SS5      R                  5       S   5      (       d   eUS   R                  S:X  d   eUS   R                  UR                  SS5      R                  5       S   5      (       d   eg )Nr   r  r  r   r|   r  rH   r_   r  r  r  r"   r^   r{   )
batch_size)r   r  rD   r   r   rd   rr   r   rX   rm   r   r   r   r  r  )r  r  r  r   rv   rX   rI  s          r   test_orc_scan_optionsr  e  s   HH288IF;88LyAC DE w#$DOOE jje,G.++G45Fv;!!9"""!9E,,.q12222.++G+BCFv;!!9"""!9EKK1-88:1=>>>>!9"""!9EKK1-88:1=>>>>r   c                       SSK Jn   g ! [         aL    [        R                  " [
        SS9   [        R                  " SSS9  S S S 5         g ! , (       d  f        g = ff = f)Nr   r8  z'not built with support for the ORC filer  rK  r  r  )r  r8  r9  r  r   r^  r   rX   r  s    r   test_orc_format_not_supportedr  |  sP    *1 *]]G
 JJs5)
 
 
 
*s&   	 #AAA
A	AAc            	         [         R                  " [        SS9   [        R                  " [
        R                  " S[        S5      05      SSS9  S S S 5        [        R                  " 5       n [         R                  " [        SS9   U R                  5         S S S 5        g ! , (       d  f       NV= f! , (       d  f       g = f)Nz9Writing datasets not yet implemented for this file formatr  r`   r   r  z/tmp)r   r  )
r  r   r+  r   r  rD   r   r4   r8  make_write_options)ofs    r   +test_orc_writer_not_implemented_for_datasetr    s    	I
 	HHc59%&uv	
	
 
			B	I
 		
 

 

 
s   6B B1 
B.1
B?c                    [         R                  " [         R                  " / SQSS9[         R                  " / SQSS9S.5      n[        U S-  5      nUR	                  5       R                  USS	9  [        R                  " U[        R                  " 5       S
9nUR                  U5      nUR                  U5      (       d   e[        U5        [        R                  " USS
9nUR                  U5      nUR                  U5      (       d   eg )Nr  rG   r|   r  rH   r_   test.csvFr+   r  r3  )rD   r   r   rd   r  to_csvr   rX   r2  r   r  r[   )r  r  r   rv   rX   rI  s         r   test_csv_formatr    s    HH288IG<88LyAC DE w#$D	OOT/jjb&6&6&89G$$W-F==/8jje,G$$W-F==r   compression)bz2gziplz4zstdc                    [         R                  R                  U5      (       d  [        R                  " U S35        [
        R                  " [
        R                  " / SQSS9[
        R                  " / SQSS9S.5      n[        R                  " 5       nUS:w  a  UOS	n[        U S
U 3-  5      nUR                  XaS9 nUR                  5       R                  SS9nUR                  UR                  S5      5        S S S 5        [         R"                  " U[         R$                  " 5       S9n	UR'                  U	5      n
U
R)                  U5      (       d   eg ! , (       d  f       N`= f)Nz support is not builtr  rG   r|   r  rH   r_   r  gzz	test.csv.r  Fr  rv  r  )r   Codecis_availabler  skiprD   r   r   rh   r   rd   rl   r  r  writer  r   rX   r2  r   r  )r  r  r  r   r
  suffixrv   r]  csv_strrX   rI  s              r   test_csv_format_compressedr    s#    ==%%k22{m#89:HH288IG<88LyAC DE##%J'61[tFw9VH--.D		&	&t	&	E //#***7

7>>'*+ 
F jjb&6&6&89G$$W-F== 
F	Es    >E
E&c           	         [        U S-  5      n[        US5       nUR                  S5        S S S 5        [        R                  " USS9nUR                  U5      nUR                  [        R                  " S[        R                  " / SQ5      05      5      (       d   e[        R                  " U[        R                  " [        R                  R                  SS	9S
9S9nUR                  U5      nUR                  [        R                  " S[        R                  " SS/5      05      5      (       d   e[        R                  " U[        R                  " [        R                  R                  S/S9S
9S9nUR                  U5      nUR                  [        R                  " S[        R                  " / SQ5      05      5      (       d   eg ! , (       d  f       GN= f)Nr  wzskipped
col0
foo
bar
r3  r  skipped)col0r  r  r"   )r'  r(  r  r  r  rw  )r  r  r  r  )rd   rQ   r  r   rX   r   r  rD   r   r   r2  r3  r5  )r  r  rv   r]  rX   rI  s         r   test_csv_format_optionsr    s   w#$D	dCD

./ 
jje,G$$W-F==
)RXX&<=>?A A A A jjb&6&6VV''!'4'6 7G$$W-F==6288UEN+C"DEFFFFjjb&6&6VV''eW'='? @G$$W-F==
%"CDEFH H H H 
s   G
Gc           
      j   [        U S-  5      n[        US5       nUR                  S5        S S S 5        [        R                  " U[        R
                  " [        R                  R                  SS9S9S9nUR                  U5      n/ SQnUR                  U:X  d   eUR                  [        R                  " [        R                  " S	/5      [        R                  " S
/5      [        R                  " S/5      [        R                  " S	/5      S.5      5      (       d   eg ! , (       d  f       N= f)Nr  r  z1,a,true,1
T)autogenerate_column_namesr(  r  )f0r+  r  r)  r"   r`   )rd   rQ   r  r   rX   r2  rD   r3  r5  r   rw  r  r   r   )r  r  rv   r]  rX   rI  expected_column_namess          r   (test_csv_format_options_generate_columnsr    s    w#$D	dCD

>" 
 jjb&6&6VV''$'G'I JG$$W-F4"7777==1#)+3%)+4&)9)+1##8 9 : : : : 
s   D$$
D2c           	         [        U S-  5      n[        US5       nUR                  S5        S S S 5        [        R                  " USS9n[
        R                  R                  S/SS9n[        R                  " U[        R                  R                  S	S
9S9nUR                  XFS9nUR                  [        R                  " S[        R                  " / SQ5      05      5      (       d   e[        R                  " US9n[        R                  " X(S9nUR                  U5      nUR                  [        R                  " S[        R                  " / SQ5      05      5      (       d   e[        R                  " 5       nUR                  XFS9nUR                  [        R                  " S[        R                  " / SQ5      05      5      (       d   eg ! , (       d  f       GN= f)Nr  r  zcol0
foo
spam
MYNULL
r3  r  MYNULLT)null_valuesr>  rA  rB  )r@  r  )fragment_scan_optionsr  )r  spamNr?  )r  r  r  )rd   rQ   r  r   rX   r   r3  rE  rD  rD   r5  r   r  r   r   r2  )	r  r  rv   r]  rX   r@  r   rI  r  s	            r   test_csv_fragment_optionsr    s   w#$D	dCD

./ 
jje,Gkk00hZEI 1 KO'''VV''5'9;G $$W$LF==62884I+J"KLMMMM!!/BJjj1G$$W-F==62884I+J"KLMMMM'')G$$W$LF==
&"((#<=>?A A A A% 
s   G
G!c                    [         R                  " [         R                  " / SQSS9[         R                  " / SQSS9S.5      n[        U S-  5      nUR	                  5       R                  SS	9S
S R                  SS5      n[        US5       nUR                  U5        S S S 5        [        R                  " U[        R                  " 5       S9nUR                  U5      nUR                  U5      (       d   e[        U5        [        R                  " USS9nUR                  U5      nUR                  U5      (       d   eg ! , (       d  f       N= f)Nr  rG   r|   r  rH   r_   	test.jsonrecordsorientr"   r]  },{}
{r  r  r7  )rD   r   r   rd   r  to_jsonreplacerQ   r  r   rX   r6  r   r  r[   r  r  r   rv   rw   r  rX   rI  s           r   test_json_formatr    s   HH288IG<88LyAC DE w$%D
//

#
#9
#
5a
;
C
CE6
RC	dCA	 
 jjb&7&7&9:G$$W-F==/8jjf-G$$W-F== 
s   E
Ec           
      4   [         R                  " [         R                  " / SQSS9[         R                  " / SQSS9S.5      n[        U S-  5      nUR	                  5       R                  SS	9S
S R                  SS5      n[        US5       nUR                  U5        S S S 5        [        R                  " [        SS9   [        R                  " U[        R                  " [         R                  R!                  SS9S9S9nS S S 5        [        R                  " U[        R                  " [         R                  R!                  SS9S9S9nUR#                  U5      nUR%                  U5      (       d   eg ! , (       d  f       N= f! , (       d  f       N= fNr  rG   r|   r  rH   r_   r  r  r  r"   r]  r  r  r  ztry to increase block sizer  r   rB  r(  r  @   )rD   r   r   rd   r  r  r  rQ   r  r  r   r^  r   rX   r6  r7  r5  r   r  r  s           r   test_json_format_optionsr    sB   HH288IG<88LyAC DE w$%D
//

#
#9
#
5a
;
C
CE6
RC	dCA	 
 
z9
;**T"*;*;,,,:+< =
;
 jjb&7&7WW((B(7'9 :G$$W-F== 

; 
;s   E8AF	8
F	
Fc                    [         R                  " [         R                  " / SQSS9[         R                  " / SQSS9S.5      n[        U S-  5      nUR	                  5       R                  SS	9S
S R                  SS5      n[        US5       nUR                  U5        S S S 5        [        R                  " [        SS9   [        R                  " [         R                  R                  SS9S9n[        R                   " U[        R"                  " U5      S9nS S S 5        [        R                  " [         R                  R                  SS9S9n[        R                   " U[        R"                  " U5      S9nUR%                  U5      nUR'                  U5      (       d   eg ! , (       d  f       GN= f! , (       d  f       N= fr  )rD   r   r   rd   r  r  r  rQ   r  r  r   r^  r   rF  r7  r5  rX   r6  r   r  )	r  r  r   rv   rw   r  r   rX   rI  s	            r   test_json_fragment_optionsr  .  s^   HH288IG<88LyAC DE w$%D
//

#
#9
#
5a
;
C
CE6
RC	dCA	 
 
z9
;,,,,,:<**T"*;*;G*DE	
; ((WW((B(79Gjjb&7&7&@AG$$W-F== 

; 
;s   F$AF6$
F36
Gc                 z   [        U S-  5      nS GH  u  p4[        US5       nUR                  U5        S S S 5        [        R                  " S[        R
                  " 5       4S[        R
                  " 5       4/5      n[        R                  " S/S/S.US	9n[        R                  R                  US
9n[        R                  " US9n	[        R                  " X)S9n
U
R                  R                  U5      (       d   eU
R                  5       R                  U5      (       a  GM   e   g ! , (       d  f       N= f)Nr  ))latin-1s   a,b
un,lphant)utf16s    a , b 
 u n ,  l  p h a n t wbr`   ra   un
   éléphantr_   rg   encodingr(  r  )rd   rQ   r  rD   rB   rI   r   r3  r5  r   r2  rX   r  r   )r  r  rv   r  
input_rowsr]  rV  rW  r  r"  dataset_transcodeds              r   test_encodingr  E  s   w#$D! $JJz"  ))c299;%7#ryy{9K$LM)5#8@OQ vv))8)<&&LAZZA!((//@@@@!**,33NCCCC%! s   D,,
D:	c                 >   [        U S-  5      n[        US5       nUR                  S5        S S S 5        [        R                  " S[        R
                  " 5       4S[        R
                  " 5       4/5      n[        R                  " S/S/S.US	9n[        R                  " US
US9n[        R                  " [        R                  R                  SS9   UR                  U5        S S S 5        [        R                  R!                  SS9n[        R"                  " US9n[        R                  " X(S9n	U	R                  R%                  U5      (       d   eU	R                  5       R%                  U5      (       d   eg ! , (       d  f       GNV= f! , (       d  f       N= f)Nr  r  s   ,b
un,lphant   éra   r  r  )r  ra   rg   r3  r  zinvalid UTF8r  r  r  r(  r  )rd   rQ   r  rD   rB   rI   r   r   rX   r  r   r   rf  rq  r   r3  r5  r2  r  )
r  r  rv   r]  rV  rW  rX   r  r"  r  s
             r   test_column_names_encodingr  ^  sB   w#$D	dD	T

/0 
 ii$		!4sBIIK6H IJOXXdV%1N4<KMN jjeODG	w{{//~	F( 
G 66%%y%9L""=KD=$$++O<<<<&&(//????% 
	 
G	Fs   E<F<
F
Fc                    SSK Jn  [        R                  " [        R                  " / SQSS9[        R                  " / SQSS9S.5      nU S	-  nUR                  5         U" U[        US
-  5      5        [        R                  " U[        R                  " 5       S9nUR                  U5      nUR                  U5      (       d   e[        U5        [        R                  " USS9nUR                  U5      nUR                  U5      (       d   eUR                  USS/S9nUR                  SS/:X  d   eUR                  USS/S9nUR                  SS/:X  d   eU" U[        US-  5      SS9  [        R                  " [         5         UR                  [        R                  " USS95        S S S 5        g ! , (       d  f       g = f)Nr   )r4  r  r   r|   r  rH   r_   feather_datasetr0  r  r1  ra   r`   r.   zdata1.featherr"   version)pyarrow.featherr4  rD   r   r   r  rd   r   rX   r,  r   r  r[   rw  r  r   r^  )r  r  r4  r   r}  rX   rI  s          r   test_feather_formatr  v  s   -HH288IF;88LyAC DE ))GMMO%W~567jj)9)9);<G$$W-F==/8jj3G$$W-F== $$WsCj$AF3*,,,$$WsCj$AF3*,,, %W67C	z	"

79 EF 
#	"	"s   %F66
G)r  r  brotlic           	         [         R                  " [         R                  " S/S-  SS9[         R                  " / SQS-  SS9S.5      n[         R                  R	                  U5      (       d  [
        R                  " 5         U S	-  nUR                  5         [        R                  " 5       nU S
-  nUR                  5         [        R                  " U[        US-  5      UUR                  S S9S9  US:X  ay  [
        R                  " [        SS9   UR                  US9nS S S 5        [
        R                  " [        SS9   [         R                  " U5      nUR                  US9nS S S 5        g UR                  US9n[        R                  " U[        US-  5      UUS9  [        R                  " U[        R                  " 5       S9n	UR!                  U	5      n
U
R#                  U5      (       d   eUS-  S-  nUR%                  5       R&                  nUS-  S-  nUR%                  5       R&                  nX:  d   eg ! , (       d  f       GN&= f! , (       d  f       g = f)Nr   ,  r   r|   r  r  rH   r_   feather_dataset_compressedfeather_dataset_uncompressedz
data.arrowr  r   file_optionsr  zCompression typer  r  part-0.arrow)rD   r   r   r  r  r  r  r  r   r,  r  rd   r  r   r^  rX   r   r  statst_size)r  r  r  r   r}  r"  uncompressed_basedirwrite_optionscodecrX   rI  compressed_filecompressed_sizeuncompressed_fileuncompressed_sizes                  r   test_feather_format_compressedr    s"    HH288QCG&988L$49EG HE88  --44GMMO""$K"%CC  </0 333E	 h]]:-?@'::' ; )M A ]]:-?@HH[)E'::u:MM A 	22{2KMGl"#"	 jj)9)9);<G$$W-F==,~=O%**,44O,|;nL)..088...1 A@ A@ 	s   H?&I?
I
Ic           	      h   / n[        S5       Hl  n[        R                  " U/S-  [        S5       Vs/ s H  n[        R                  " 5       PM     snS.5      n[        R
                  " U[        U 5      US9  Mn     [        U S-  5      n[        R                  " WR                  UUS9  XT4$ s  snf )zG
Creates a simple (flat files, no nested partitioning) Parquet dataset
r   r   r  metadata_collector	_metadata)	r4   rD   r   r  rq   r  rd   write_metadatarB   )	root_pathr  r?   r  r   metadata_paths         r   _create_parquet_dataset_simpler    s    
 1Xb%PR)0T)Q)0TUV
3y>6H	
  	K/0Mm-  1Us   B/c                 &   U S-  n[        U5      u  p#[        R                  " U5      nUR                  R	                  UR                  5      (       d   e[        UR                  5      S:X  d   eUR                  5       nUR                  S:X  d   eg )Nr  r   (   )	r  r   parquet_datasetrB   r  r   r  r   r   )r  r  r  r   rX   rI  s         r   test_parquet_dataset_factoryr    s     00I9)DM  /G>>  ....w}}"""F??b   r   win32z'Results in FileNotFoundError on Windows)reasonc                    [         R                  " S5      nU S-  n[        U5      u  p4UR                  S5      n[        R
                  " [        R                  " U5      5      n[        R                  " X6S9nUR                  R                  UR                  5      (       d   e[        UR                  5      S:X  d   eUR                  5       nUR                  S:X  d   eg )NrU  r  rV  r  r   r	  )r  rC  r  r
  rh   r   r?  r   r
  rB   r  r   r  r   r   )	r  rU  r  r  r   r[  r
  rX   rI  s	            r   #test_parquet_dataset_factory_fsspecr    s       *F 00I9)DM !!&)I !1!1)!<=J  FG>>  ....w}}"""F??b   r   c                    U S-  n[         R                  " S/S-  [        R                  R	                  S5      S.5      n/ n[
        R                  " U[        U5      US9  [        US-  5      n[
        R                  " UR                  UUS9  [        R                  " U5      nUR                  R                  UR                  5      (       d   eUR                  5       nUR                  S:X  d   eg )Nr  r   r   r  r  r  )rD   r   rr  r  randnrq   r  rd   r  rB   r   r
  r  r   r   )r  r  r   r  r  rX   rI  s          r   &test_parquet_dataset_factory_roundtripr    s     00IHHQC"HBIIOOB,?@AEs9~2D 	K/0Mm-   /G>>  ....F??b   r   c                    / n[        S5       Hj  n[        R                  " S[        [        US-  US-   S-  5      5      05      nX S3-  n[        R
                  " X4US9  US   R                  U S35        Ml     [        U S-  5      n[        R                  " WR                  XQ5        [        R                  " U5      nUR                  5       nUR                  S5      R                  5       nU[        [        SS	5      5      :X  d   eg )
Nr   r+  r"   r]   r  r]  r  r   r  )r4   rD   r   rm   rq   rr   set_file_pathrd   r  rB   r   r
  r   r=  rx  )	r  	metadatasr?   r   
table_pathr  rX   scanned_tablescanned_cols	            r   "test_parquet_dataset_factory_orderr     s     I 2Y4adQqS"H-./1H~-

uYG"##qcN3  +-.MellM=  /G$$&M&&t,668K$uQ}----r   c                    U S-  n[        U5      u  p#[        UR                  S5      5      S   R                  5         [        R
                  " U5      nUR                  R                  UR                  5      (       d   e[        UR                  5      S:X  d   e[        R                  " [        5         UR                  5         S S S 5        g ! , (       d  f       g = f)Ntest_parquet_dataset_invalid	*.parquetr   r   )r  rm   globunlinkr   r
  rB   r  r   r  r  r   r-  r   )r  r  r  r   rX   s        r   $test_parquet_dataset_factory_invalidr  6  s     88I9)DM	$%a(//1  /G>>  ....w}}"""	(	) 
*	)	)s   5C
Cc                    [        [        U R                  S5      5      5      n[        R                  " US   5      R
                  R                  5       n/ nU H]  n[        R                  " U5      R                  nUR                  [        UR                  U 5      5      5        UR                  U5        M_     U S-  n[        R                  " X&US9  U$ )Nr  r   r  r  )rm   rC  rglobrq   r  rB   to_arrow_schemar  r  rd   r  r5   r  )r  parquet_pathsrB   r  rv   r  r  s          r   _create_metadata_filer$  D  s    	 <=>M^^M!,-44DDFF>>$'00s4#3#3I#>?@!!(+ 
 +M2D r   c           
         [         R                  " [         R                  " [        S5      5      [         R                  " [        R
                  R                  S5      5      [         R                  " [        R                  " SS/S5      5      // SQS9nUR                  SS05      n[        R                  " U[        U 5      S	/S
9  [        U 5      U4$ )Nr  r`   ra   r   r  r  r   r,   r   r  )rD   r   r   r4   rr  r  r  rq  rL   rq   r  rd   r$  )r  r   s     r   #_create_parquet_dataset_partitionedr&  V  s    HH
rRXXbiioob&9:
C:r*+- #E
 ))5'*:;Es9~vhG +U22r   c                     U S-  n[        U5      u  p#[        R                  " SS9n[        R                  " X$S9nUR                  R                  UR                  5      (       d   e[        UR                  5      S:X  d   eUR                  5       nUR                  S:X  d   eUR                  5       R                  S5      R                  SS	9nUR                  5       n[        R                  R                  Xg5        g )
N(test_parquet_dataset_factory_partitionedr   r  r  r{   r  r+  Tdrop)r&  r   r   r
  rB   r  r   r  r   r   r  sort_valuesreset_indexr8   testingassert_frame_equal)r  r  r  r   r   rX   rI  r  s           r   r(  r(  a  s     DDI>yIM??&1L  JG>>  ....w}}"""F??b    ++D1==4=HF HJJ!!&3r   c                 R   U S-  n[        U5      u  p#[        R                  " USS9nUR                  R	                  UR                  5      (       d   eSUR                  R
                  ;   d   e[        UR                  5       5      nSUS   R                  R
                  ;   d   eg )N%test_parquet_dataset_factory_metadatar   r     keyr   )	r&  r   r
  rB   r  r  rm   rP   r  )r  r  r  r   rX   r$  s         r   r0  r0  u  s     AAI>yIM  VDG>>  ....W^^,,,,,W**,-IYq\11:::::r   c                    Uu  p#U S-  n[        U5      u  pVU" U/5         [        R                  " U[        R                  " SS9US9nS S S 5        U" / 5         [	        WR                  5       5      nS S S 5        U" / 5         [	        WR                  [        R                  " S5      S:  5      5        S S S 5        U" / 5         WS   R                  [        R                  " S5      S:  5        S S S 5        U" / 5         WS   R                  5       n	U	S   R                  5         S S S 5        g ! , (       d  f       N= f! , (       d  f       N= f! , (       d  f       N= f! , (       d  f       Ny= f! , (       d  f       g = f)N#test_parquet_dataset_lazy_filteringr   r  )r   r
  r+     r   )	r  r   r
  r   rm   rP   rE   r  r  )
r  r   rh   r   r  r  r  rX   r$  rg_fragmentss
             r   r3  r3    s9    'B
 ??I5i@M 
}o	&$$7 
' 
b	..01	 
 
b	W""288D>B#678 
 
b	!''(;< 
 
b	 |668Q002 
	' 
'	& 
	 
	 
	 
	s;   )D1E2E,E$'E51
D?
E
E!$
E25
Fc                 N   [         R                  " S/ SQ05      nU S-  nUR                  U5        [        R                  " U5      nUR                  U5      R                  nUR                  US/S9R                  nSUR                  ;   d   eUR                  USS9(       d   eg )Nr`   r  r  r.   s   pandasTrL  )	r8   r9   r  r   rX   r   rB   r  r  )r  r  rM   rv   rX   rB   rs  s          r   test_dataset_schema_metadatar7    s     
sI&	'B^#DMM$jjG$$W-44F%..w.FMM '''==)$=???r   c                    [         R                  " S[         R                  " / SQSS905      n[        R                  " U[        U S-  5      5        [         R                  " S[         R                  " 5       4/5      n[        R                  " U S-  SUS9nUR                  U[        R                  " S5      S:  S	9nUS   R                  US   R                  S
5      R                  S5      5      (       d   e[        UR!                  5       5      S   nUR                  U[        R                  " S5      S:  US9nUS   R                  US   R                  S
5      R                  S5      5      (       d   eg )Nr  r&  r   r|   rK  r   r  r{   r  rG   r   r  )rD   r   r   rq   rr   rd   rB   rG   r   rX   r   rE   r  r  r  rm   rP   )r  r  r   rB   rX   filteredrY   s          r   test_filter_mismatching_schemar:    sB    HHeRXXlABCENN5#g678 YY
+,-Fjj. 6CG
 &&wrxx7J&KHE?!!%,"3"3G"<"B"B1"EFFFFG))+,Q/H&&%1,V ' =HE?!!%,"3"3G"<"B"B1"EFFFFr   c                    [         R                  " SR                  5       [        [	        S5      5      S.5      n[        U S-  5      n[        R                  " X#S/S9  [        R                  " USS9nUR                  U5      nUR                  US/S	9nUR                  S5      R                  UR                  S5      5      (       d   eg )
Nza a b br   r*  rK  r   r  r   r  r.   )rD   r   rR  rm   r4   rd   rq   r  r   rX   r   r=  r  )r  r  r   rv   rX   all_cols	part_onlys          r   +test_dataset_project_only_partition_columnsr>    s     HHioo/U1XGHEw'(DVH=jjF3G&&w/H''&'BI??6")))*:*:6*BCCCCr   c           	         [         R                  " S[        R                  " / SQSS905      nU S-  nUR	                  USS9  [
        R                  " US[        R                  " S[        R                  " 5       4/5      S	9n[        R                  " S[        R                  " / SQ[        R                  " 5       5      05      nUR                  U5      R                  U5      (       d   eg )
Nr  r  objectdtypez(test_dataset_project_null_column.parquetr   r  r   r  )r8   r9   rr  r   r  r   rX   rD   rB   rG   r   r   r  )r  r  rM   r  rX   r  s         r    test_dataset_project_null_columnrC    s     
ubhh'9JK	LB<<AMM!IM&jj9 "		E288:+>*? @BGxx);RXXZ HIJH""7+228<<<<r   c           	         SSK Jn  [        R                  " / SQ/ SQ/ SQS.5      nUR	                  X0S-  5        [
        R                  " U S-  SS	9nUR                  U[
        R                  " S
5      [
        R                  " S5      R                  SSS9[
        R                  " S5      S:H  S.S9n[        R                  " / SQ[        R                  " / SQSS9/ SQS.5      nUR                  U5      (       d   e[        R                  " [        SS9   UR                  US
S
0S9  S S S 5        g ! , (       d  f       g = f)Nr   r1  r  )r1  r8  r9  rw  r  r0  r1  r  r  r  r   Fsafer  r`   )	A_renamedB_as_intC_is_ar.   r|   )TFFzExpected an Expressionr  )r   r1  rD   r   r4  r   rX   r   rE   r  r   r  r  r   r!  )r  r  r1  r   rX   rI  r  s          r   test_dataset_project_columnsrK    s   HH9<oNOE%>!9:jj>1)DG$$WXXc]HHSM&&wU&;((3-3&7$ F
 xxHHYW5& H
 =="""" 
y(@	A#s< 
B	A	As   D44
Ec                 D   [        U 5      u  p[        R                  " U5      n[        UR                  [        R
                  5      (       d   e[        U 5      u  pB[        R                  " U5      n[        UR                  [        R
                  5      (       d   e[        R                  " USS9nUR                  nUc   e[        U[        R                  5      (       d   eUR                  [        R                  " S[        R                  " 5       4/5      :X  d   e[        UR                  5      S:X  d   eUR                  S   [        R                  " / SQ[        R                  " 5       5      :X  d   e[        R                  " [        R                  " S[        R                  " 5       4/5      SS9n[        U[        R                  5      (       d   e[        UR                  5      S:X  d   e[        S UR                   5       5      (       d   e[        R                  " X%S9nUR                  n[        U[        R                  5      (       d   eUR                  [        R                  " S[        R                  " 5       4/5      :X  d   e[        UR                  5      S:X  d   e[        S	 UR                   5       5      (       d   e[        R                  " USS9n[        R                  " [!        UR#                  5       5      UR                  UR$                  UR&                  S
9nUR                  b   eU S-  n[)        U5      u  p[        R*                  " USS9nUR                  nUc   e[        U[        R                  5      (       d   eUR                  [        R                  " S[        R,                  " 5       4/5      :X  d   e[        UR                  5      S:X  d   e[/        UR                  S   R1                  5       5      SS1:X  d   eg )Nr   r  r   r"   r   )r   r"   r{   r  c              3   (   #    U  H  oS L v   M
     g 7fr   r    r  s     r   r  6test_dataset_preserved_partitioning.<locals>.<genexpr>+       4"3QDy"3r  c              3   (   #    U  H  oS L v   M
     g 7fr   r    r  s     r   r  rN  2  rO  r  r  zdata-partitioned-metadatar`   ra   )r3  r   rX   rR   r   r   r  r  rB   rD   r   r   r  r   r  r  rm   rP   r   r
  r&  r
  rI   r   rx  )	r  r  rv   rX   r  r   r  r  r  s	            r   #test_dataset_preserved_partitioningrQ    sW    "'*GAjjGg**B,D,DEEEE 37;JjjGg**B,D,DEEEE jjF3GDdB//0000;;"))fbhhj%9$:;;;;t  !Q&&&Q288Irxxz#BBBB ??299vrxxz&:%;<VLDdB//0000t  !Q&&&4$"3"344444jj1GDdB//0000;;"))fbhhj%9$:;;;;t  !Q&&&4$"3"344444 jjF3G##W""$%gnn~~'*<*<H   ((( 55I:9EM  VDGDdB//0000;;"))fbiik%:$;<<<<t  !Q&&& t  #--/0S#J>>>r   c                    [         R                  " [         R                  " S[         R                  " 5       5      [         R                  " S[         R                  " [         R
                  " 5       [         R                  " 5       5      5      /5      n[         R                  " / SQ[        [        S5      5      S.US9n[        U S-  5      n[        R                  " X#S/S9  [        R                  " U S-  5      nUR                  S5      R                  5       UR                  S5      R                  5       :X  d   eUR                  S5      R!                  UR                  S5      5      (       d   eg )	Nr  r   )NNr`   r`   r   r*  rg   rK  r  )rD   rB   rE   rG   r   r   rI   r   rm   r4   rd   rq   r  rt  r=  rx  r  )r  rB   r   rv   actual_tables        r   +test_write_to_dataset_given_null_just_worksrT  J  s    YY

#
rxxz299;?@ F HH4!%(^-5;=E w'(DVH===>!9:L v&00 	f		'	'	)* * *u%,,U\\%-@AAAAr   c                 x    SS K Jn  UR                  " XR                  " US4/5      S9nUR                  " X5      $ )Nr   	ascending)r   )pyarrow.computecomputesort_indicesSortOptionsr   )tabsort_colr  sorted_indicess       r   _sort_tabler^  _  s7     __^^h%<$=>@N773''r   c                 l   U=(       d    Un[         R                  " XSUSS9  [        UR                  S5      5      n[	        U5      [	        U5      :X  d   e[         R
                  " USUS9n[        UR                  5       U5      R                  [        U R                  5       U5      5      (       d   eg )Nr  Fr   r   r   *r  )	r   r  rm   r!  r   rX   r^  r   r  )rX   r  expected_filesr\  base_dir_pathr   
file_pathsr  s           r   _check_dataset_roundtripre  f  s    !-XMWw".EC m))#./Jz?c.1111 zzgLBH x((*H5<<G$$&13 3 3 3r   c                    U S-  nUR                  5         [        U5      n[        R                  " U5      nU S-  nUS-  /n[	        U[        U5      USU5        U S-  nUS-  /n[	        X4USU5        U S-  nUS-  /n[        U 5         [	        USUSU5        S S S 5        U S-  nUR                  5         [        U5      n[        R                  " U5      nU S	-  nUS-  /n[	        U[        U5      USU5        g ! , (       d  f       Nh= f)
Nr  zsingle-file-targetr  r`   zsingle-file-target2zsingle-file-target3z./single-file-target3r  zsingle-directory-target)r  r3  r   rX   re  rd   r
   r  )r  ru   r  rX   targetrb  s         r   test_write_datasetrh  y  s&    -'IOOI&Ajj#G ++F~-.NWc&k>3O ,,F~-.NWnc6J ,,F~-.N	G	 ,nc6	K 

 ,,IOO"9-Ajj#G00F~-.NWc&k>3O 
	s   	C;;
D	c           	      B   U S-  n[        U5      n[        R                  " SS9n[        R                  " XS9nU S-  nUS-  US-  S-  US-  US-  S-  /n[        R                  " [        R
                  " S	[        R                  " 5       4/5      SS9n[        U[        U5      US
UUS9  U S-  nUS-  US-  S-  US-  US-  S-  /n[        R                  " [        R
                  " S	[        R                  " 5       4/5      5      n[        U[        U5      US
UUS9  g )Npartitionedr   r  r  zpartitioned-hive-targetpart=ar  part=br   r+  partitioned-dir-targetr`   ra   )	r&  r   r   rX   rD   rB   rI   re  rd   )r  ru   r  r   rX   rg  expected_pathsr  s           r   test_write_dataset_partitionedro    s4    -'I+I6A??&1Ljj>G 00F6H,~=6H,~=N //
		FBIIK()*6;VndF(*
 //Ffsl^3fsl^3N //
		FBIIK()*,VndF(*r   c                    [         R                  " / SQ/ SQS.5      n[        R                  " XSS/S9  [        R                  " U SS/S9nUR
                  nU Vs1 s H;  n[        [        R                  " U5      R                  U 5      R                  5      iM=     nnU1 Sk:X  d   eUR                  5       nUR                  U5      (       d   eg s  snf )Nrh  rj  r_   r  ra   r  >   r#  ri  r  rD   r   r   r  rX   r  rd   r  r  r  r  r   r  r  r   r  r  r  partitioning_dirsr  s          r   #test_write_dataset_with_field_namesrt    s    HH+/BCEUE#&%) 

75uEIOOEBGBGQGLLO''0778%   ///((*O!!%((((   ACc                    [         R                  " / SQ/ SQS.5      n[        R                  " XSS/SS9  [        R                  " U SSS9nUR
                  nU Vs1 s H;  n[        [        R                  " U5      R                  U 5      R                  5      iM=     nnU1 S	k:X  d   eUR                  5       nUR                  U5      (       d   eg s  snf )
Nrh  rj  r_   r  ra   r   )r   r   partitioning_flavorr  >   b=xb=yb=zrq  rr  s          r   (test_write_dataset_with_field_names_hiver{    s    HH+/BCEUE#&%VE 

75vFIOOEBGBGQGLLO''0778%    5555((*O!!%((((ru  c                    [         R                  " / SQ/ SQ/ SQS.5      n[        R                  " XSS/S9  [        R                  " U SS/S9n[
        R                  " 5        n[        R                  " UR                  SS/S	9USS/S9  [        R                  " USS/S9nUR                  5       n[        UR                  5       5      UR                  S
5      R                  5       :X  d   e S S S 5        g ! , (       d  f       g = f)Nrh  rj  r  rw  r  ra   r  r   r.   r`   )rD   r   r   r  rX   r  r  r   r   r  rB  drop_columnsr  r   rX   tempdir2r  r  s         r   test_write_dataset_with_scannerr    s    HH+/$& 'E UE#&%) jjcUCG		$	$	&(
#s<!%se	E JJxSEJ	#,,.O--/ ++C0::<= 	= = 
'	&	&s   $BC99
Dc           	        ^ ^	^
^^^^^^ [         R                  " 5       m " U4S jS[        5      n[        R                  " U" [        R
                  " 5       5      5      m[        R                  " [        R                  " S[        R                  " 5       5      /5      n[        R                  " [        R                  " [        [        S5      5      5      /US9m	Sm
SnSmS	mU	U
UU4S
 jn[        R                  R!                  U" 5       US	S9m[         R"                  " UUU 4S jS9nUR%                  5          [&        R&                  " 5       mU4S jnSnSnU" 5       S:  a4  T
U:  a  T
U:X  a  S	nO%T
n[&        R(                  " S5        U" 5       S:  a  M4  U(       d   e SmTR+                  5         UR-                  5         g ! SmTR+                  5         UR-                  5         f = f)Nc                   "   > \ rS rSrU 4S jrSrg)6test_write_dataset_with_backpressure.<locals>.GatingFsi  c                 V   > TR                  5         U R                  R                  XS9$ )Nr  )waitr   rl   )r   rv   r  consumer_gates      r   rl   Itest_write_dataset_with_backpressure.<locals>.GatingFs.open_output_stream  s'     88..t.GGr   r    N)r   r   r   r   rl   r   )r  s   r   GatingFsr    s    	H 	Hr   r  r>   r  rg   r          Tc               3   z   >#    TT:  a0  T(       d  g [         R                  " S5        TS-  mT v   TT:  a  M/  g g 7f)Ng{Gz?r"   )r  sleep)ry   batches_readend
keep_goings   r   counting_generator@test_write_dataset_with_backpressure.<locals>.counting_generator  s9     S JJtALK S s   4;;r  c                  D   > [         R                  " T[        T5      ST S9$ )Nr   r8  )r   r  rd   )	gating_fsr   r  s   r   r9  6test_write_dataset_with_backpressure.<locals>.<lambda>  s    r''S\)	Kr   )rg  c                  4   > [         R                   " 5       T -
  $ r   )r  )starts   r   duration6test_write_dataset_with_backpressure.<locals>.duration  s    99;&&r   Fr   r  )	threadingEventr   rh   r   r   rD   rB   rE   r   ro   r   rm   r4   r   rp  rp   Threadr  r  r  r   r  )r  r  rB   min_backpressurer  write_threadr  
last_valuebackpressure_probably_hitry   r  r  r  r  r  r   r  s   `        @@@@@@@@r   $test_write_dataset_with_backpressurer    s    OO%M
H< H
 ););)= >?IYY456F OORXXd5+;&<=>vNEL
CJ  jj%%V & ?G ##KLL !			' 
$)!j2o..:-04-)
JJsO j2o )((
 
 
s   ;AG 	G $G*c                    [         R                  " / SQ/ SQS.5      n[        R                  " XSS/S9  [        R                  " U SS/S9n[
        R                  " 5        n[        R                  " X#SS/S9  [        R                  " USS/S9nUR                  5       n[        UR                  5       5      UR                  5       :X  d   e S S S 5        g ! , (       d  f       g = f)Nrj  r  ra   r   r  ra   r  )
rD   r   r   r  rX   r  r  r   r  rB  r~  s         r   test_write_dataset_with_datasetr  @  s    HH?;<EUE#&%) jjcUCG		$	$	&(
 %SE	; JJxSEJ	#,,.O--/0EOO4EEEE 
'	&	&s   !A,C
C%c           	      ^   U S-  n[         R                  " / SQ/ SQS.5      n[        R                  " [         R                  " [         R
                  " S[         R                  " 5       5      /5      SS9nS n[        R                  " X!US	S
9  [         R                  " / SQ/ SQS.5      n[        R                  " [         R                  5         [        R                  " X!US	S
9  S S S 5        [         R                  " SS/05      nUS-  S-  n[        R                  R                  XV5        [        R                  " X!US	SS9  [         R                  " / SQ/ SQS.5      n[        R                  " U S	US9R                  5       nU" X5        UR!                  5       (       d   e[        R                  " X!US	SS9  [         R                  " / SQ/ SQS.5      n[        R                  " U S	US9R                  5       nU" X5        UR!                  5       (       a   eg ! , (       d  f       GN>= f)Nr   rj  r  r  r   r   )rB   r  c                     U R                  5       R                  S5      R                  SS9nUR                  5       R                  S5      R                  SS9nUR                  U5      (       d   eg )Nra   Tr)  )r  r+  r,  r  )r  r  df1df2s       r   compare_tables_ignoring_orderGtest_write_dataset_existing_data.<locals>.compare_tables_ignoring_orderX  s\    lln((-99t9Dlln((-99t9Dzz#r   r  r+  rw  r  ra   ezc=2z	foo.arrowoverwrite_or_ignore)r   r   existing_data_behavior)r  r#  r`   ra   r   )r{   r"   r{   r^   r   r  delete_matching)r#  r`   ra   r   r&  )rD   r   r   r   rB   rE   rG   r  r  r   rq  r   r1  r4  rX   r   exists)	r  ru   r   r   r  extra_table
extra_fileoverwrittenreadbacks	            r    test_write_dataset_existing_datar  Q  s   $IHH?;<E??"))	#rxxz	"#+%-35L ULOHH?;<E 
r	'
&25	B 
( ((C#<(KU"[0JOO!!+: UL!,AC (('o>@Kzz'%'355=XZ !(8 UL!:KM ((!5LIJKzz'%'355=XZ !(8  """""9 
(	's   H
H,c                 l    [        U 5       Vs/ s H  n[        R                  " X5      PM     sn$ s  snf r   )r4   r  randint)re  r  r  r  s       r   _generate_random_int_arrayr    s'    .3Dk:kFNN3$k:::s    1c           	          / n/ n[        U 5       H:  nUR                  [        USUS95        UR                  S[        U5      -   5        M<     [        R
                  " X#S9nU$ )Nr"   )re  r  r  r   r>   r  )r4   r5   r  rd   rD   ro   )num_of_columnsnum_of_recordsr>   rw  r?   ro   s         r   _generate_data_and_columnsr    sg    DL>".N343AC 	D 	C#a&L)	 #
 ??ALr   c                 v    [        [        [        R                  " U 5      R	                  SU 35      5      5      $ )Nz**/*.)r   rm   r  r  r  base_directoryr"  s     r   _get_num_of_files_generatedr    s-    tGLL055k]6KLMNNr   c                 
  ^ U S-  nSmSnSnSn[        UU5      n[        R                  " XQSTUS9  [        R                  " U5      nUT-  S-   n[        U5      U:X  d   e/ n[        U5       HT  u  pU[        U
5      -  n[        R                  " USS9nUR                  UR                  5       R                  S	   5        MV     U[        U5      :X  d   eU[        U5      :X  d   e[        U4S
 jU 5       5      (       d   eg )Nr   r   r{   #   r   )r   max_rows_per_filemax_rows_per_groupr"   r  r   c              3   .   >#    U  H
  nUT:*  v   M     g 7fr   r    )r  file_rowcountr  s     r   r  7test_write_dataset_max_rows_per_file.<locals>.<genexpr>  s      <$:=  11$:s   )r  r   r  ry  rz  r   rj   rd   rX   r5   r   shaper  r  )r  ru   r  r  r  ro   files_in_direxpected_partitionsresult_row_combinationr  f_filef_pathrX   r  s                @r   $test_write_dataset_max_rows_per_filer    s+   $INN-n.<>L \Y'8(:< ::i(L ),==A | 3333  |,	S[(**VI6%%g&6&6&8&>&>q&AB - #&<"====S!78888 <$:< < < < <r   c                    U S-  nSnSnSn/ SQnU Vs/ s H  n[        UU5      PM     nnUS-  n[        R                  " XxUUSS9  [        R                  " U5      n	[        U	5       H  u  pU[        U5      -  n[        R                  " USS	9nUR                  5       nUR                  5       n[        U5       H<  u  nnUR                  nU[        U5      S
-
  :  a  UU:  a  UU::  d   eM4  UU::  a  M<   e   M     g s  snf )Nr   r  rb  r{   )
r#   r#   r#   r#   r#   r   r   r   r   r   min_rows_groupr   )r>   r  min_rows_per_groupr  r   r  r"   )r  r   r  ry  rz  rj   rd   rX   r   r   r   r   )r  ru   r  r  r  record_sizesr  record_batchesdata_sourcer  r  r  r  rX   r   batchesr  ry   rows_per_batchs                      r   %test_write_dataset_min_rows_per_groupr    s/   $IN1L -9:,8. 11?A,8  : ..K.(:(:%'
 ::k*L|,	s6{***VI6  """$"7+IB"^^NCL1$$%);;"&889 98%);;;; , -:s   Dc                    U S-  nSnSnSn[        UU5      nUS-  n[        R                  " XVUSS9  [        R                  " U5      n/ nU Hj  n	U[        U	5      -  n
[        R                  " U
SS9nUR                  5       nUR                  5       nU H  nUR                  UR                  5        M      Ml     USS	/:X  d   eg )
Nr   r  r{      max_rows_groupr   )r>   r  r  r   r  ry  )r  r   r  ry  rz  rd   rX   r   r   r5   r   )r  ru   r  r  r  ro   r  r  batched_datar  r  rX   r   r  ry   s                  r   %test_write_dataset_max_rows_per_groupr    s    $INN-n.<>L ..K,(:%' ::k*LLs6{***VI6  """$E/   B8###r   c           	      |   U S-  nSnSnSS/n[         R                  " / SQ/ SQ/US9n[         R                  " / S	Q/ S
Q/US9n[         R                  " / SQ/ SQ/US9n[         R                  " / SQ/ SQ/US9n[         R                  R                  XVXx/5      n	[        R
                  " [         R                  " XC   [         R                  " 5       4/5      SS9n
US-  n[        R                  " XXS9  S nU" XUU5      u  pX:X  d   eUS-  nSn[        R                  " XXUSS9  U" XUU5      u  pX:  d   eg )Nr   r   r"   c1c2)r"   r{   r^   r   r   r   )r`   ra   r   r   r  r`   r  )r#   r  re  rb  r   r"   )r`   ra   r   r   r  r   )r  r   rz  ry  r   r"   )r`   ra   r   r   r  r   )r  r/  r4  ra  r   r"   )r`   ra   r   r   r  ra   r   r  default)r>   r  r   r   c                 n    [        XS9n[        [        R                  R	                  X   5      5      nXE4$ )Nr  )r  r   rD   rX  unique)r  ro   r"  col_idnum_of_files_generatednumber_of_partitionss         r   _get_compare_pair<test_write_dataset_max_open_files.<locals>._get_compare_pair  s7    !<&"A"2::#4#4\5I#JK%;;r   max_1r^   F)r>   r  r   r   max_open_filesr   )	rD   ro   rJ   rp   r   r   rB   rI   r  )r  ru   r"  partition_column_idrw  record_batch_1record_batch_2record_batch_3record_batch_4r   r   data_source_1r  r  r  data_source_2r  s                    r   !test_write_dataset_max_open_filesr    s   $IK$<L__+>+I+K+79N __+=+I+K+79N __+@+I+K+79N __+A+I+K+79N HH!!>#1#C DE ??
		L5ryy{CDEL 	)M%".D< M;/1 1 "999 'MN%".$2G
 M;/1 1 "888r   c           	         U S-  n[        U5      n[        R                  " U[        R                  R	                  SS9S9nU S-  nUS-  US-  S-  US-  US-  S-  /n[        R
                  " [        R                  " UR                  R                  S	5      /5      S	[        R                  " SS/5      0S
9n[        U[        U5      USUUS9  g )Nrj  TrH  r  rm  r`   r  ra   r   r  r+  )r&  r   rX   r  r?  r   rD   rB   rE   r   re  rd   )r  ru   r  rX   rg  rn  r   s          r   #test_write_dataset_partitioned_dictr  A  s     -'I+I6A jj((1141HJG //Ffsl^3fsl^3N ??299V$.& $'bhhSz235L VndF!#r   c           	        ^ U S-  n[        U5      n[        R                  " USS9n[        R                  " [        R
                  " S[        R                  " 5       4/5      SS9nU S-  n/ mU4S jn[        R                  " X5SUS	US
9  US-  S-  US-  S-  1n[        [        [        R                  T5      5      nX:X  d   eU S-  n	[        R                  " X9SUSS9  [        R                  " USUS9n
[        R                  " U	SUS9nU
R                  5       R                  UR                  5       5      (       d   eg )Nrj  r   r  r   r  partitioned1c                 <   > TR                  U R                  5        g r   )r5   rv   )written_filepaths_writtens    r   file_visitor4test_write_dataset_use_threads.<locals>.file_visitorh  s    \../r   r1  Tr   r   r   r  rk  part-0.featherrl  partitioned2Fr`  r  )r&  r   rX   r   rD   rB   rI   r  r   rn   r  r  r   r  )r  ru   r  rX   r   target1r  rn  paths_written_settarget2result1result2r  s               @r   test_write_dataset_use_threadsr  [  s?    -'I+I6Ajj8G??
		FBIIK()*6;L &GM0 | 	(--(--N Cm<=...&G jjNGjjNG$$W%5%5%78888r   c                 H   [         R                  " S[        S5      05      nUR                  SS9n[        R
                  " X SSSS9  [        R                  " U 5      R                  SS	9S   R                  5       nS
nU H  n[        U5      nXd:  d
   SU 35       eUnM!     g )Nr`   r  r{   )max_chunksizer   T)r   r   preserve_orderFr   r]  z!Sequence expected to be ordered: )
rD   r   r4   r   r   r  rX   r   to_numpyr  )r  r   r  seqprevitemcurrs          r   -test_write_dataset_use_threads_preserve_orderr    s     HHc5;'(EQ/GWi!%d<
**W

&
&5
&
9#
>
G
G
ICD4y{E?uEE{ r   c           	        ^^ [         R                  " [         R                  " [        S5      5      [         R                  " S [        S5       5       5      [         R                  " S/S-  S/S-  -   5      // SQS9nU S-  n[        R
                  " XS	S
S9  [        UR                  S5      5      nUS-  /n[        U5      [        U5      :X  d   e[        R                  " USS9R                  5       nUR                  U5      (       d   eU S-  nUS-  US-  S-  US-  US-  S-  /n/ m/ mUU4S jn[        R                  " [         R                  " S[         R                  " 5       4/5      SS9n[        R
                  " XS
S	XvS9  [        UR                  S5      5      n[        U5      [        U5      :X  d   eT Vs/ s H"  n[        R                   R#                  U5      PM$     n	nTU	:X  d   e[        R                  " USUS9nUR                  5       R                  U5      (       d   e[%        T5      S:X  d   eT H  n
[&        R(                  " U
5      U;   a  M   e   g s  snf )Nr  c              3   L   #    U  H  n[         R                   " 5       v   M     g 7fr   r  r  s     r   r  #test_write_table.<locals>.<genexpr>  r  r  r`   r   ra   r  r  singledat_{i}.arrowr1  basename_templater   ra  zdat_0.arrowr  r  rj  rk  rl  c                 r   > TR                  U R                  5        TR                  U R                  5        g r   )r5   rv   re  )r  visited_pathsvisited_sizess    r   r  &test_write_table.<locals>.file_visitor  s+    \../\../r   r   r   r  )r   r  r   r  r  r{   )rD   r   r   r4   r   r  rm   r!  r   rX   r   r  r   rB   rI   ry  rv   getsizer   r  r  )r  r   r  rd  rn  rI  r  r   rv   actual_sizesvisited_pathr  r  s              @@r   test_write_tabler    s<   HH
rRXX%IuRy%II
#sebj() "#E
 !HU'6yJ hnnS)*J./Nz?c.1111ZZ/88:F== &H8X0=@8X0=@N
 MM0 ??
		FBIIK()*6;LUY'6".K hnnS)*Jz?c.11116CDmdBGGOOD)mLDL(((ZZ\JF??##E****}"""%||L)^;;; & Es   )I)c           	         [         R                  " [         R                  " [        S5      5      [         R                  " S [        S5       5       5      [         R                  " S/S-  S/S-  -   5      // SQS9n[         R                  " U/S-  5      nU S	-  n[
        R                  " XS
S9  [        UR                  S5      5      [        US-  /5      :X  d   e[
        R                  " USS9R                  5       R                  U5      (       d   eU S-  n[
        R                  " U/US
S9  [        UR                  S5      5      [        US-  /5      :X  d   e[
        R                  " USS9R                  5       R                  U5      (       d   eU S-  n[
        R                  " UR                  5       US
S9  [        UR                  S5      5      [        US-  /5      :X  d   e[
        R                  " USS9R                  5       R                  U5      (       d   eU S-  n[
        R                  " X/US
S9  [        UR                  S5      5      [        US-  /5      :X  d   e[
        R                  " USS9R                  5       R                  [         R                  " U/S-  5      5      (       d   eg )Nr   c              3   L   #    U  H  n[         R                   " 5       v   M     g 7fr   r  r  s     r   r  6test_write_table_multiple_fragments.<locals>.<genexpr>  r  r  r`   r#   ra   r  r  r{   r  r1  r  ra  r  r  zsingle-listmultiplezmultiple-table)rD   r   r   r4   r  r   r  r   r!  rX   r   r  r   )r  r   r  s      r   #test_write_table_multiple_fragmentsr     sx   HH
rRXX%IuRy%II
#cUQY&' "#E eWQY'E !HUY7x~~c"#sH7G,G+H'IIII::hu-668??FFFF &HeWhy9x~~c"#sH7G,G+H'IIII::hu-668??FFFF #HU%%')Dx~~c"#s	$	$%(' ' ' '::hu-668??FFFF ))He^Xi@x~~c"#s	$	$%(' ' ' '::hu-668??
%#   r   c           	         [         R                  " [         R                  " [        S5      5      [         R                  " S [        S5       5       5      [         R                  " S/S-  S/S-  -   5      // SQS9nU S-  n[        R
                  " S	 UR                  5        5       UUR                  S
SS9  [        R                  " USS9R                  5       nUR                  U5      (       d   eU S-  n[         R                  R                  UR                  UR                  5       5      n[        R
                  " XBS
SS9  [        R                  " USS9R                  5       nUR                  U5      (       d   eU S-  n[        U5      n[        R
                  " XRS
SS9  [        R                  " USS9R                  5       nUR                  U5      (       d   eg )Nr  c              3   L   #    U  H  n[         R                   " 5       v   M     g 7fr   r  r  s     r   r  &test_write_iterable.<locals>.<genexpr>  r  r  r`   r   ra   r  r  inmemory_iterablec              3   $   #    U  H  ov   M     g 7fr   r    )r  ry   s     r   r  r#    s     <);e);s   r  r  )rB   r  r   r  inmemory_readerr  inmemory_pycapsule)rD   r   r   r4   r   r  r   rB   rX   r   r  r  rp   r   )r  r   r  rI  r  streams         r   test_write_iterabler)    s   HH
rRXX%IuRy%II
#sebj() "#E
 ,,H<)9)9);<h!LL'6uF ZZ/88:F==**H!!..u||/4/?/?/ACFVQVWZZ/88:F==--H&FVQVWZZ/88:F==r   c           	         [         R                  " [         R                  " [        S5      5      [         R                  " S [        S5       5       5      [         R                  " S/S-  S/S-  -   5      // SQS9n[        R
                  " U5      nU S-  n[        R                  " UR                  U5      US	S
9  UR                  [        R
                  " USS
95      nUR                  U5      (       d   eU S-  n[        R                  " UR                  US/S9US	S
9  UR                  [        R
                  " USS
95      nUR                  UR                  S/5      5      (       d   e[        R                  " [        SS9   [        R                  " UR                  U5      UUR                  S	S9  S S S 5        g ! , (       d  f       g = f)Nr  c              3   L   #    U  H  n[         R                   " 5       v   M     g 7fr   r  r  s     r   r  %test_write_scanner.<locals>.<genexpr>  r  r  r`   r   ra   r  r  dataset_from_scannerr1  r  r  dataset_from_scanner2r+  r.   zCannot specify a schemar  )rB   r   )rD   r   r   r4   r   rX   r  r   r   r  r  r  r   r^  rB   )r  r  r   rX   r  rI  s         r   test_write_scannerr/    s~   HH
rRXX%IuRy%II
#sebj() "#E jjG//H^++9.$$RZZ%GHF== 00H^++GdV+Di1$$RZZ%GHF==tf-.... 
z)B	C
//8( %Y	@ 
D	C	Cs   1G  
Gc                    [         R                  " [         R                  " [        S5      5      [         R                  " S/S-  S/S-  -   5      R	                  5       /SS/S9n[
        R                  " UR                  S/5      R                  5      nU S-  n[
        R                  " XS	US
9  [
        R                  R                  S/SS9n[
        R                  " USUS
9R                  5       nUR                  U5      (       d   eg )Nr  r`   r   ra   r  r   r  rX   r1  r  TrH  r  )rD   r   r   r4   r  r   r   r  rB   r  r   r?  rX   r   r  )r  r   r   r  partitioning_readrI  s         r   !test_write_table_partitioned_dictr2    s     HH
r
#sebj();;= V_E
 ??5<<#9#@#@AL"H	
 0099	4 : )ZZ->hj  ==r   c                 L   [         R                  " [         R                  " [        S5      SS9[         R                  " [        R
                  " SSSS9R                  S5      5      [         R                  " [        R                  " SS	/S
5      5      // SQS9nU S-  n[        R                  " XSS9  [        UR                  S5      5      nUS-  /n[        U5      [        U5      :X  d   e[        R                  " USS9R                  5       nUR                  U5      (       d   eS GHv  n[        R                   " 5       nUR#                  US9nS[%        U5      ;   d   eU SU 3-  n[        R                  " XXxS9  [&        R(                  " US-  5      n	US:X  a  SOSn
U	R*                  U
:X  d   e[        R                  " USS9R                  5       nUR,                  nUS:X  aC  UR                  SUR/                  S5      R1                  [         R2                  " 5       5      5      nUS;   aD  UR                  SUR/                  S5      R1                  [         R4                  " S5      5      5      nUR7                  U5      nUR                  U5      (       a  GMw   e   g )Nr  r  r|   r_  zdatetime64[D]rA  zdatetime64[ns]r`   ra   r   r  r  r
  r   r  ra  part-0.parquet)1.02.42.6r  z(<pyarrow.dataset.ParquetFileWriteOptionsparquet_dataset_versionr  r5  r7  r   )r5  r6  r"   r  )rD   r   r   r4   rr  rs  r  rq  r   r  rm   r!  r   rX   r   r  r   r  r2  rq   read_metadataformat_versionrB   rE   	with_typerG   r  r  )r  r   r  rd  rn  rI  r  r   optsmetaexpected_versionrB   r  s                r   test_write_dataset_parquetr?  3  s<    HH
r*
<?CJJ 	
C:r*+	
 "#E **HUY7hnnS)*J!112Nz?c.1111ZZ3<<>F== )%%'(((99T$ZGGG6wi@@
K+; ;<$+u$45%""&6666 HY7@@BeZZ6<<?#<#<RXXZ#HIFn$ZZ6<<?#<#<R\\$=O#PQF::f%}}X&&&&' )r   c           	      b   [         R                  " [         R                  " [        S5      5      [         R                  " S [        S5       5       5      [         R                  " S/S-  S/S-  -   5      // SQS9nU S-  n[        R
                  " XS	S
9  [        UR                  S5      5      nUS-  /n[        U5      [        U5      :X  d   e[        R                  " US	S
9R                  5       nUR                  U5      (       d   e[        R                  " [        R                  R                  UR                   R"                  S9S9nUR%                  SS9nU S-  n[        R
                  " XXgS9  [        R                  " X&S
9R                  5       nUR                  U5      (       d   eg )Nr  c              3   L   #    U  H  n[         R                   " 5       v   M     g 7fr   r  r  s     r   r  )test_write_dataset_csv.<locals>.<genexpr>b  r  r  r`   r   ra   )r+  r  chr1r  csv_datasetr3  r  ra  z
part-0.csvr  r(  F)include_headercsv_dataset_noheaderr  )rD   r   r   r4   r   r  rm   r!  r   rX   r   r  r2  r   r3  r5  rB   r  r  )r  r   r  rd  rn  rI  r   r<  s           r   test_write_dataset_csvrG  `  sm   HH
rRXX%IuRy%II
#sebj() "#E
 &HUU3hnnS)*J-.Nz?c.1111ZZ/88:F== 7;;+B+B\\'' ,C ,) *F$$E$:D//HUVGZZ099;F==r   c           	      R  ^ [         R                  " [         R                  " [        S5      5      [         R                  " S [        S5       5       5      [         R                  " S/S-  S/S-  -   5      // SQS9nSmU4S	 jnU S
-  n[        R
                  " XSUS9  T(       d   eg )Nr  c              3   L   #    U  H  n[         R                   " 5       v   M     g 7fr   r  r  s     r   r  :test_write_dataset_parquet_file_visitor.<locals>.<genexpr>}  r  r  r`   r   ra   r  r  Fc                 \   > U R                   b  U R                   R                  S:X  a  Smg g g )Nr^   T)r  rf  )r  visitor_calleds    r   r  =test_write_dataset_parquet_file_visitor.<locals>.file_visitor  s2    !!-%%11Q6!N 7 .r   r
  r   )r   r  )rD   r   r   r4   r   r  )r  r   r  r  rL  s       @r   'test_write_dataset_parquet_file_visitorrN  z  s    HH
rRXX%IuRy%II
#sebj() "#E
 N" **HUY".0 >r   c           	        ^^ [        S5       VVs/ s H  o/S-    H  o"PM     M     nnn[        S5       VVs/ s H  o/S-    H  o"S-  PM	     M     nnn[        R                  " X4S/S-  S/S-  -   S.5      nU S-  n[        R                  " [        R
                  " S[        R                  " 5       4/5      S	S
9n/ mS mUU4S jn[        R                  " XVSUSUS9  US-  S-  US-  S-  1n	[        [        [        R                  T5      5      n
X:X  d   eTc   eTR                  S:X  d   eg s  snnf s  snnf )Nr   r   r`   r  ra   r  rj  r   r   r  c                 v   > U R                   (       a  U R                   mTR                  U R                  5        g r   )r  r5   rv   )r  r  sample_metadatas    r   r  Atest_partition_dataset_parquet_file_visitor.<locals>.file_visitor  s+      *33O\../r   r   Tr  rk  r4  rl  r{   )r4   rD   r   r   r   rB   rI   r  r   rn   r  r  rf  )r  r   r
  f1_valsf2_valsr   r  r   r  rn  r  r  rQ  s              @@r   +test_partition_dataset_parquet_file_visitorrU    sK   !&qCgldtltGC$)!HFH52BwwHGFHHG"ebjC52:57 8E -'I??
		FBIIK()*6;L MO0 | 	H//H//N Cm<=...&&&&&!+++? DFs
   D5D;c                 4   [         R                  " S[        R                  " SSS9/05      nUS   R                  R
                  S:X  d   e[        R                  " XSS9  [        R                  " U S-  5      nUS   R                  R
                  S:X  d   eg )Nr`   r_  zEurope/Brussels)tzr   r  r4  )
rD   r   r8   ra  r}   rW  r   r  rq   rt  )r  r   rI  s      r   (test_write_dataset_arrow_schema_metadatarX    s    
 HHcBLL:KLMNOE:??!2222UI6]]7%556F#;"3333r   c                     SSK Jn  [        R                  " S/ SQ05      nUR	                  SS05      n[
        R                  " X SS9  UR                  U S	-  5      R                  nUR                  SS0:X  d   eg )
Nr   rE  r`   r  r1     valuer1  r  r  )
r   r1  rD   r   rL   r   r  rt  rB   r  )r  r1  r   rB   s       r   "test_write_dataset_schema_metadatar[    sp    HHc9%&E))68*<=EUI6*: :;BBF??vx0000r   c                     [         R                  " S/ SQ05      nUR                  SS05      n[        R                  " XSS9  [
        R                  " U S-  5      R                  nUR                  SS0:X  d   eg )Nr`   r  r1  rZ  r   r  r4  )	rD   r   rL   r   r  rq   rt  rB   r  )r  r   rB   s      r   *test_write_dataset_schema_metadata_parquetr]    sm     HHc9%&E))68*<=EUI6]]7%556==F??vx0000r   c           	         U u    pppEnSR                  XVX45      n[        R                  " [        R                  " [	        S5      5      [        R                  " S [	        S5       5       5      [        R                  " S/S-  S/S-  -   5      // SQS9n[
        R                  " [        R                  " S	[        R                  " 5       4/5      S
S9n	[
        R                  " USUSU	S9  [
        R                  " SUSS
S9R                  5       n
U
R                  U5      (       d   eUR                  S5      n[
        R                  " XSU	S9  [
        R                  " SUSS
S9R                  5       n
U
R                  U5      (       d   eUR                  S5      n[
        R                  " USUSU	S9  [
        R                  " SUSS
S9R                  5       n
U
R                  U5      (       d   eg )NrJ  r  c              3   L   #    U  H  n[         R                   " 5       v   M     g 7fr   r  r  s     r   r  (test_write_dataset_s3.<locals>.<genexpr>  r  r  r`   r   ra   r  r  r   r   r  zmybucket/datasetr1  rt  r  zmybucket/dataset2r  r4  r  zmybucket/dataset3)r   rD   r   r   r4   r   r   rB   rI   r  rX   r   r  )rh  r  rh   rj  rk  rl  rm  uri_templater   r   rI  ri  s               r   test_write_dataset_s3rb    s    7H3Aqat=DDD	0 
 HH
rRXX%IuRy%II
#sebj()+ #E
 ??299vryy{&;%<=fMD !b
 ZZr%fhj  == 

1
2CU	EZZ5vhj  == 

j
)Czc)$ ZZ5vhj  ==r   aC  {
    "Version": "2012-10-17",
    "Statement": [
        {
            "Effect": "Allow",
            "Action": [
                "s3:PutObject",
                "s3:ListBucket",
                "s3:GetObjectVersion"
            ],
            "Resource": [
                "arn:aws:s3:::*"
            ]
        }
    ]
}c           
      ~   SSK Jn  U S   u  p#  n[        U [        SS5        U" SSU SU 3SS9n[        R
                  " [        R                  " [        S	5      5      [        R                  " S
 [        S	5       5       5      [        R                  " S/S-  S/S-  -   5      // SQS9n[        R                  " [        R                  " S[        R                  " 5       4/5      SS9n[        R                  " USUSSUSS9  [        R                  " SUSSS9R                  5       nUR                  U5      (       d   e[        R                  " USUSSUSS9  [        R                  " SUSSS9R                  5       nUR                  U5      (       d   e[         R"                  " [$        SS9   [        R                  " USUSSSS9  S S S 5        U" SSU SU 3SSS 9n[         R"                  " [$        S!S9   [        R                  " USUSSSS9  S S S 5        g ! , (       d  f       NY= f! , (       d  f       g = f)"Nr   )rD  r1  test_dataset_limited_user
limited123r3  http)rl  rm  endpoint_overrideschemer  c              3   L   #    U  H  n[         R                   " 5       v   M     g 7fr   r  r  s     r   r  1test_write_dataset_s3_put_only.<locals>.<genexpr>2  r  r  r`   r   ra   r  r  r   r   r  zexisting-bucketr1  Fr  )r
  r   rk   r   r  r  rt  Tz&Bucket 'non-existing-bucket' not foundr  znon-existing-bucket)r
  r   rk   r  limited)rl  rm  rg  rh  allow_bucket_creationz(Access Denied|ACCESS_DENIED))r   rD  r   _minio_put_only_policyrD   r   r   r4   r   r   rB   rI   r  rX   r   r  r  r   rg  )	r6  rD  rj  rk  r  rh   r   r   rI  s	            r   test_write_dataset_s3_put_onlyrn    s8    ( !.D1y*@:LJ	.!F!D6*	
B HH
rRXX%IuRy%II
#rSEBJ&') #E
 ??299vryy{&;%<=fMD  RU4 ZZbVhj  ==  RT4 ZZbVhj  == 
wE
G
(R#8	

G 
!F!D6*"
B 
w&E	F
(R#8	
 
G	F!
G 
G  
G	Fs   0H:H.
H+.
H<c           
         [         R                  " SS S /05      n[        R                  " X S-  5        [         R                  " [         R
                  " S[         R                  " [         R                  " 5       [         R                  " 5       5      5      /5      n[        R                  R                  U S-  /U[        R                  " 5       [        R                  " 5       S9nUR                  U5      nUR                  U:X  d   eg )Nr`   r  )r   rB   r   r
  )rD   r   rq   rr   rB   rE   r   r   rI   r   r  r  r   rh   r   r   )r  r  r   rB   fsdss        r   $test_dataset_null_to_dictionary_castrq  i  s     HHcD$<()ENN5N23YY
bmmBHHJ		<= F **'(##%%%'	 + D ##D)E<<6!!!r   c                 p   [         R                  " / SQ/ SQS.5      n[        R                  " XS-  SS9  [        R                  " U S-  SS9n[         R                  " / SQ/ SQS	.5      n[        R                  " X0S
-  SS9  [        R                  " U S
-  SS9nUR                  USS5      nUR                  5       [         R                  " / SQ/ SQ/ SQS.5      :X  d   eUR                  USSSS9nUR                  5       R                  S5      [         R                  " / SQ/ SQ/ SQS.5      :X  d   eg )Nr"   r{   r  r`   ra   r  colAr  r  r  r  c   r{   r"   Zr  r  )colBcol3r  rv  r{  r  r  Nrv  r  r|  
full outer)	join_typer"   r{   r  rx  r`   ra   r  Nr  r  Nrz  rD   r   r   r  rX   r  r   rA  r  r  ds1r  ds2rI  s         r   test_dataset_joinr  |  s!   	 
B R46
**Wt^E
2C	 
B R46
**Wt^E
2CXXc66*F?? * !    XXc66\XBF??$$V,%%: 1   r   c                 n   [         R                  " / SQ/ SQS.5      n[        R                  " XS-  SS9  [        R                  " U S-  SS9n[         R                  " / SQ/ SQS	.5      n[        R                  " X0S
-  SS9  [        R                  " U S
-  SS9nUR                  US5      nUR                  5       [         R                  " / SQ/ SQ/ SQS.5      :X  d   eUR                  USSSS9nUR                  5       R                  S5      [         R                  " / SQ/ SQ/ SQS.5      :X  d   eg )Nrs  rt  ru  r  r  r  rw  ry  )rv  r|  r  rv  r}  r~  r  _rr  right_suffixr  r  r  r  r  s         r   test_dataset_join_unique_keyr    s   	 
B R46
**Wt^E
2C	 
B R46
**Wt^E
2CXXc6"F?? * !    XXc6\XMF??$$V,%%: 1   r   c           	         [         R                  " / SQ/ SQ/ SQS.5      n[        R                  " XS-  SS9  [        R                  " U S-  SS9n[         R                  " / SQ/ S	Q/ S
QS.5      n[        R                  " X0S-  SS9  [        R                  " U S-  SS9nUR                  USSSS9nUR                  5       R                  S5      [         R                  " / SQ/ SQ/ SQ/ SQ/ SQ// SQS9:X  d   eg )Nrs  r   r  <   rt  )rv  r{  colValsr  r  r  rw  rx  r  r   ry  r  rv  r  r  r  r  )r   r  r  Nr  )r   r  Nrx  r  )rv  r{  r  colB_r	colVals_rr  r  r  s         r   test_dataset_join_collisionsr    s    	" 
B
 R46
**Wt^E
2C	" 
B
 R46
**Wt^E
2CXXc6\XMF??$$V,: @1A A A Ar   c           	         [         R                  R                  / SQ/ SQS.5      n[        R                  " XS-  SS9  [        R
                  " U S-  SS9n[         R                  R                  / SQ/ SQ/ S	QS
.5      n[        R                  " X0S-  SS9  [        R
                  " U S-  SS9nUR                  USSSSSS9nUR                  5       R                  S5      [         R                  " / SQ/ SQ/ SQS.5      :X  d   eg )N)r"   r"   r#   r  re  )r`   ra   r`   ra   r  ru  r  r  r  )r{   r  r4  )r`   ra   g)r1  r9  g      @)r{  r|  colCr  rv  r  r"   r{  r|  onby	toleranceright_onright_by)r1  NNNN)rv  r  r  )
rD   rJ   from_pydictr   r  rX   	join_asofr   rA  r   r  s         r   test_dataset_join_asofr    s    			) 
B R46
**Wt^E
2C			 
B
 R46
**Wt^E
2C]]6Q&  F ??$$V,),: 1   r   c                    [         R                  " / SQ/ SQ/ SQS.5      n[        R                  " XS-  SS9  [        R                  " U S-  SS9n[         R                  " / SQ/ S	Q/ S
Q/ SQS.5      n[        R                  " X0S-  SS9  [        R                  " U S-  SS9nUR                  USSS/SS9nUR                  5       R                  S5      [         R                  " / SQ/ SQ/ SQ/ SQS.5      :X  d   eg )Nrs  r  r  )rv  r{  r  r  r  r  r  ry  rw  r  )r{  r  rv  r  r  r  rv  r{  r"   r  r  r  )Nr  Nrv  r{  r  r  )rD   r   r   r  rX   r  r   rA  r  s         r   "test_dataset_join_asof_multiple_byr    s    	 
B
 R46
**Wt^E
2C	"	 
B R46
**Wt^E
2C]]&&)Q  F ??$$V,$	: 1   r   c                    [         R                  " S/ SQ05      n[        R                  " XS-  SS9  [        R                  " U S-  SS9n[         R                  " / SQ/ SQS.5      n[        R                  " X0S	-  SS9  [        R                  " U S	-  SS9nUR                  US/ S
S9nUR                  5       [         R                  " / SQ/ SQS.5      :X  d   eg )Nr  r  r  r  r  ry  r  )r  r  r  r"   r  )rz  rz  r  )r  r  )rD   r   r   r  rX   r  r   r  s         r   test_dataset_join_asof_empty_byr    s    	i 
B R46
**Wt^E
2C	" 
B R46
**Wt^E
2C]]q  F ??"* !   r   c           
         [         R                  " / SQ/ SQ/ SQ/ SQS.5      n[        R                  " XS-  SS9  [        R                  " U S-  SS9n[         R                  " / S	Q/ S
Q/ SQ/ SQ/ SQS.5      n[        R                  " X0S-  SS9  [        R                  " U S-  SS9nSn[
        R                  " [        US9   UR                  USSS/SSSS/S9  S S S 5        g ! , (       d  f       g = f)Nrs  r  r  rt  r  r  r  r  r  ry  )r  r  r  rw  r  )r{  r  colUniqrv  r  r  zXColumns {'colVals'} present in both tables. AsofJoin does not support column collisions.r  r  rv  r{  r"   r  )	rD   r   r   r  rX   r  r   r^  r  )r  r  r  r  r  ry  s         r   !test_dataset_join_asof_collisionsr  (  s    	"	 
B R46
**Wt^E
2C	"" 
B R46
**Wt^E
2C	7  
z	-Dff-VV$4 	 	
 
.	-	-s   C''
C5dstyperh   memc           	         [         R                  " / SQ/ SQS.5      nUS:X  a1  [        R                  " X S-  SS9  [        R                  " U S-  SS9nO#US:X  a  [        R                  " U5      nO[
        eUR                  [        R                  " S	5      S
:  5      R                  [        R                  " S5      S:H  5      nUS:X  a  [        R                  O[        R                  n[        XE5      (       d   eUR                  5       [         R                  " S/S/S.5      :X  d   eUR                  S5      [         R                  " S/S/S.5      :X  d   eUR                  [        R                  " S	5      S:  5      R                  [        R                  " S	5      S:  5      R                  [        R                  " S	5      S:g  S9nUR                  5       [         R                  " S/S/S.5      :X  d   e[        R                  " X@S-  SS9  [        R                  " U S-  SS9nUR                  5       [         R                  " S/S/S.5      :X  d   eUR                  [        R                  " [         R                  " SS/SS/S.5      5      SSS9nUR                  5       R!                  S5      [         R                  " SS /SS/SS/S.5      :X  d   e["        R$                  " [&        5         UR                  S 5        S S S 5        ["        R$                  " [(        5         UR+                  5         S S S 5        UR,                  R/                  S5      n	UR                  [        R                  " S	5      S
:  5      R1                  U	5      n
U
R                  5       [         R                  " S	SS/05      :X  d   e["        R$                  " [         R2                  5         UR1                  U	5      R                  5         S S S 5        g ! , (       d  f       GN= f! , (       d  f       N= f! , (       d  f       g = f)Nr"   r{   r  rb  r`   ra   r  r  ru  rh   r  r  r  r  rv  r^   r  r`   r"   r#   rb  r  r  r{   ra   r9  r   r  r{  r  zright outerkeysr  r{  )rv  r{  r  )rD   r   r   r  rX   r+  r  r  rE   r  r  rR   r   r   r   r  rA  r  r   r!  r^  rP   rB   r  replace_schemarq  )r  r  r  r  rI  r  r2r9  joinedschema_without_col2	newschemas              r   test_dataset_filterr  H  s    
$ 
B ~
t^E:jj46	5jjn!! ZZ(1,-44RXXf5E5LMF'-~r##2;M;MHf''''??* !   
 ;;q>RXX'     
BHHV$q(	)	0	0
1
%gRXXf-=-BgC ;;=BHH&     Vz1%@zz'J.u=H"((, #    [[BHHRc
. %    /F ??$$V,D	Rc
: 1    
y	!

4 
" 
z	" 
# **++A.


1n()  288A- $    
r	' 	12;;=	 
(	' 
"	! 
#	" 
(	's$   :P.P&+ P7
P#&
P47
Qc           	         [         R                  " / SQ/ SQS.5      n[         R                  " / SQ/ SQS.5      nUS:X  aa  [        R                  " X S-  SS	9  [        R                  " U S-  SS	9n[        R                  " X0S
-  SS	9  [        R                  " U S
-  SS	9nO9US:X  a-  [        R                  " U5      n[        R                  " U5      nO[
        e[        R                  " XE45      R                  [        R                  " S5      S:  [        R                  " S5      S:H  -  5      nUR                  5       [         R                  " / SQ/ SQS.5      :X  d   eUR                  [        R                  " [         R                  " SS/SS/S.5      5      SSS9nUR                  5       R                  S5      [         R                  " / SQ/ SQ/ SQS.5      :X  d   eUR                  [        R                  " S5      S:  5      nUR                  [        R                  " S5      S:  5      n	[        R                  " [        SS9   [        R                  " X45        S S S 5        g ! , (       d  f       g = f)Nr  r  ru  )r  r   rz  )hr?   lrh   r  r  r  r  r  rv  r^   r  )r"   r{   r  )r`   ra   r  r   r  r`   ra   r  r  z
left outerr  )r   r  N)rv  r  r{  zcurrently not supportedr  )rD   r   r   r  rX   r+  r  r  rE   r   r  rA  r  r   r^  )
r  r  r  r  r  r  filtered_union_dsr  filtered_ds1filtered_ds2s
             r   test_union_dataset_filterr    s    
$ 
B 
 
B ~
t^E:jj46
t^E:jj46	5jjnjjn!!

C:.55	&	A	"((6"2a"78 %%'2885 ,   
 ##BJJrxxRc
9 0 %  $ .F ??$$V,: 1    ::bhhv.23L::bhhv.34L	z)B	C


L/0 
D	C	Cs   I00
I>c                    U S-  n[        U5      u  p#[        R                  " U5      nUR                  5       nUR                  S:X  d   eUR                  [        R                  " S5      S:  5      nUR                  5       R                  S:X  d   e[        R                  " [        5         UR                  5         S S S 5        g ! , (       d  f       g = f)Ntest_parquet_dataset_filterr	  r+  r{   r  )r  r   r
  r   r   r  r  rE   r  r   r^  rP   )r  r  r  r  rX   rI  filtered_dss          r   r  r    s    77I5i@M  /GF??b   ..$!!34K!**b000	z	"!!# 
#	"	"s   -C
Cc                    [         R                  " [         R                  " [        S5      5      /S/S9n[        R
                  " U5      nS[        R                  " S5      0nUR                  US9n[        R                  " X@S/SS9  [        R                  " [        SS	9   [        R                  " X@S/SS9  S
S
S
5        g
! , (       d  f       g
= f)z{
Ensure the projected schema is used to validate partitions for scanner

https://issues.apache.org/jira/browse/ARROW-17228
r  original_columnr  renamed_columnr.   r  r+  z0'Column original_column does not exist in schemar  N)rD   r   r   r4   r   rX   rE   r   r  r  r   KeyError)r  r   table_datasetr/   r   s        r   4test_write_dataset_with_scanner_use_projected_schemar    s     HHbhhuRy)*3D2EFEJJu%M"((#45G ##G#4G(8'9%I 	O	
 	,=+>u	
		
 	
 	
s   "C
Cr   )r  r   c           
         US:X  a  [         R                  " S5        [        R                  " SS/SS/S SSS/SS	0S /S
.SS/ SQSS0/S
./S.5      n[        R
                  " X S-  US9  [        R                  " U S-  US9nUR                  / SQS9nUR                  5       SS SS/S S	S.S /SS.SS/ SQSS S./SS./:X  d   eg )Nr   zpyarrow.parquetabc123qrs456r"   r{   buttonr  r  )r}   elementvaluesstructsscrollwindow)Nr^   r   fizzbuzz)user_ida.dotted.fieldinteractionr   r  )r  zinteraction.typezinteraction.valueszinteraction.structsr  r.   )r  r  )r  r}   r  r  r  )	r  rC  rD   r   r   r  rX   r   rx  )r  r   r   r  s       r   test_read_table_nested_columnsr    s   -.HH(H!5)*A(q6u~t&<	>hff-=,>	@& E Ug-f=
**Ww&v
6C LL:  ;E ??dq!f"5148A	Oh,$T23q	J!   r   c                    SSK Jn  U S-  n[        R                  R	                  [        R
                  " / SQ[        R                  " 5       5      [        R
                  " / SQ[        R                  " 5       5      /SS/5      nUR                  " UUSS/S	S
9  UR                  " USS	[        R                  " [        R                  " S[        R                  " 5       5      [        R                  " S[        R                  " 5       5      /5      S9R                  5       R                  5       nX4R                  S5      :X  d   eUR                  S5      R                  5       n[!        [#        U5      5      nU Vs/ s H  nS[%        USS9-   PM     nn[!        [&        R(                  " W5      5      nXg:X  d   eg s  snf )Nr   )rX   zslash-writer-xr"   r{   r^   r   r#   )experiment/A/f.csvzexperiment/B/f.csvr  zexperiment/C/k.csvzexperiment/M/i.csvexp_idexp_metar  r   )r>   r  r   r   rw  )r  r   r   rB   r"   z	exp_meta=r  rF  )r   rX   rD   rJ   r   r   r   r  r  rB   rE   r   rI  rA  r=  rx  rC  r   r   ry  rz  )tmpdirr   rv   dt_tablert  r  encoded_pathsrd  s           r   !test_dataset_partition_with_slashr    sk   %$$Dxx##
"((*-
 ()+	4%5 8@6L	NH  \" yy"((8RXXZ8((:rwwy9; <	 hj!  ))(3333q!++-Hc(m$HDLMHD[5B#77HMM

4()J&&& Ns   ;F;c           	      `   [         R                  " [         R                  " S[         R                  " 5       SS9[         R                  " S[         R                  " 5       SS9/5      n/ SQ/ SQ/n[         R                  R                  X!S9n[        R                  " X0S	-  5        [        R                  " U S	-  S
S9nUR                  5       R                  R                  U5      (       d   e[        R                  " X0S-  S
S9  [        R                  " U S-  S
S9nUR                  5       R                  R                  U5      (       d   e[        R                  " X3/U S-  S
S9  [        R                  " U S-  S
S9nUR                  5       R                  R                  U5      (       d   eg )Nr#  F)nullableri  Tr  Nr#   Nrg   	nulltest1r   r  	nulltest2	nulltest3)rD   rB   rE   rG   rJ   r   rq   r  r   rX   r   r  r  )r  schema_nullabler  r   rX   s        r   'test_write_dataset_preserve_nullabilityr  2  sX    ii
bhhj51
bhhj40!2 3O )FHH   @E45jj;.yAG$$++O<<<<Uk1)Djj;.yAG$$++O<<<<e^W{%:9Mjj;.yAG$$++O<<<<r   c                 r   [         R                  " [         R                  " S[         R                  " 5       SS0S9[         R                  " S[         R                  " 5       5      /5      n[         R                  " [         R                  " S[         R                  " 5       5      [         R                  " S[         R                  " 5       5      /5      n/ SQ/ SQ/n[         R                  R                  X1S9n[         R                  R                  X2S9n[        R                  " XE/U S	-  S
S9  [        R                  " U S	-  S
S9nUR                  5       R                  R                  USS9(       d   e[        R                  " XT/U S-  S
S9  [        R                  " U S-  S
S9nUR                  5       R                  R                  USS9(       d   e[        R                  " XT/U S-  S
US9  [        R                  " U S-  S
S9nUR                  5       R                  R                  USS9(       d   eg )Nr#  s   foos   barr  ri  r  r  rg   test1r   r  TrL  test2test3r  )rD   rB   rE   rG   rJ   r   r   r  rX   r   r  )r  schema_metadataschema_no_metar  r   table_no_metarX   s          r   *test_write_dataset_preserve_field_metadatar  J  s   ii
bhhjFF+;<
bhhj!!# $O YY
bhhj!
bhhj! # $N )FHH   @EHH(((GM e+Ww->yQjj7*9=G$$++OD+QQQm+Ww->yQjj7*9=G$$++N4+PPP m+Ww->y+-jj7*9=G$$++OD+QQQr   c                    S GHa  nS GHV  n[         R                  " [         R                  " S[         R                  " 5       5      [         R                  " S[         R                  " 5       5      /5      n/ SQ/ SQ/n[         R                  R                  XCS9n[        R                  " 5       nU SU 3-  n[        R                  " UUSUR                  UUS	9S
S9  [        R                  " USS9nUR                   H]  n	[        R                  " U	5      n
U
R                  S5      R                  S5      nUR                   UL d   eUR"                  X!-  L a  M]   e   GMY     GMd     g )N)TFr#  ri  r  r  rg   write_page_index_r   )write_statisticswrite_page_indexr  )r   r  r  r  r   )rD   rB   rE   rG   rJ   r   r   r   r  r  rX   r  rq   r9  r  r=  has_offset_indexhas_column_index)r  r  r  rB   r  r   r"  r  r  rV  r  ccs               r   #test_write_dataset_write_page_indexr  g  sE   ) -YYbhhj)bhhj) + ,F  1FHH(((?E..0K#45E4F!GGH (;;%5%5 <  (=	 **Xi8C		++D1''*11!4**.>>>>**.>.QQQQ "- !. *r   c                     [         R                  " [         R                  " / SQ5      [         R                  " / SQ5      /SS/S9nUS:X  a1  [        R                  " X S-  SS	9  [        R
                  " U S-  SS	9nO#US
:X  a  [        R
                  " U5      nO[        eUR                  S5      R                  5       R                  5       / SQ/ SQS.:X  d   eUR                  S/5      R                  5       R                  5       / SQ/ SQS.:X  d   eUR                  [        R                  " S5      S:  5      R                  S5      R                  5       R                  5       / SQ/ SQS.:X  d   e[         R                  R                  [         R                  " / SQ[         R                  " 5       S9[         R                  " / SQ5      /SS/S9n[        R
                  " U5      nUR                  S/5      nUR                  5       R                  5       nUS   / SQ:X  d   eUS   / SQ:X  d   eUR                  S/5      nUR                  5       R                  5       nUS   / SQ:X  d   eUS   / SQ:X  d   eg )N)r^   r"   r   r{   r#   )ra   r`   ra   r`   r   r  r  r  rh   r  r  r  r  )r`   r`   ra   ra   r   r  )r  r  )r  
descending)r   ra   ra   r`   r`   )r#   r   r^   r{   r"   r   )r`   r`   ra   r  )r#   re  re  r  r|   )r  carr  foobarr`   ra   )r`   r  )r  re  re  r#   )r  r  r  r  )r`   rV  )rD   r   r   r   r  rX   r+  rA  r   rB  r  r  rE   rJ   r   rG   )r  r  r   r   
sorted_tabsorted_tab_dicts         r   test_dataset_sort_byr    sH    HH
!
*+ !E
 ~
$u=ZZ$u5	5ZZ!!::h((*446)!;   
 ::/01::<FFH)!M   
 99bhhx(1,.66hj!   HH  
RXXZ0
01" 3Z ! E 
E	B012J ))+557O3=0003#BBBB/01J ))+557O3=0003#BBBBr   c                 0   [         R                  " S/ SQ05      n[         R                  R                  5       nUR	                  SS9nU S-  n[
        R                  " UUUUS9  [
        R                  " SS9n[         R                  R                  US9n[
        R                  " UUS	9R                  5       nX:X  d   eU S
-  n[        XH5        [        UR                  5       5      n	[        U	5      S:X  d   eU	S   n
[        U
R                  5       5      nUS   US   :w  d   eUS   US   sUS'   US'   U
R                  U5        [
        R                  " SS9n[         R                  R                  US9n[
        R                  " XS	9R                  5       nX:w  d   eU[         R                  " S/ SQ05      :X  d   e[         R"                  " [$        SS9   [
        R                  " UUS	9R                  5       nSSS5        g! , (       d  f       g= f)zsCheck that checksum verification works for datasets created with
ds.write_dataset and read with ds.dataset.to_tabler`   r&  T)write_page_checksumcorrect_dir)r>   r  r   r  r  )default_fragment_scan_optionsr  corrupted_dirr"   r      $   F)r"   r^   r{   r   zCRC checksum verificationr  N)rD   r   rX   r   r  r   r  r  r   r   rm   iterdirr   	bytearray
read_byteswrite_bytesr  r   rg  )r  
table_origpq_write_formatr  original_dir_pathpq_scan_opts_crcpq_read_format_crctable_checkcorrupted_dir_pathcorrupted_file_path_listcorrupted_file_pathbin_datapq_scan_opts_no_crcpq_read_format_no_crctable_corruptr  s                   r   1test_checksum_write_dataset_read_dataset_to_tabler    s%    3-.J jj224O#66  7 "M  -/""	 44#')55&6 6 8**! hj  $$$ !?23
  $$6$>$>$@A'(A---215,779:H B<8B<'''!)"x|HRL(2, ##H- 77#(*JJ88&9 9 ;JJ::B(* 
 &&&BHHc<%89999 
w&A	BJJ%
 (* 	
 
C	B	Bs   $H
Hc                     Sn Sn[         R                  " [        5       n[        R                  R
                  R                  S5        S S S 5        U [        WR                  5      ;   d  U[        UR                  5      ;   d   e[        R                  R                  5       nSn[         R                  " [        US9   UR                  S5        S S S 5        g ! , (       d  f       N= f! , (       d  f       g = f)NzImake_write_options() should be called on an instance of ParquetFileFormatzqdescriptor 'make_write_options' for 'pyarrow._dataset_parquet.ParquetFileFormat' objects doesn't apply to a 'int'+   z;make_write_options\(\) takes exactly 0 positional argumentsr  )	r  r   r!  rD   rX   r   r  rd   r,   )msg_1msg_2excinfopformatry  s        r   test_make_write_options_errorr    s    -E(E 
y	!W


$$77; 
"C&&%3w}}3E*EEEjj**,G
IC	y	,""2& 
-	, 
"	! 
-	,s   *C>C*
C'*
C8c                 :    SS K Jn  SnSnU R                  WR                  R                  U5      UR                  R                  U5      S9R                  5       nUR                  5       SSS/0:X  d   eg ! [         a    [        R                  " S5         Nf = f)Nr   zsubstrait NOT enableds   
SOhttps://github.com/apache/arrow/blob/main/format/substrait/extension_types.yaml	
u64
	u32

str"i
i64
f64
str
const
struct
a
b
group
key7
:
Z
b
:

:
b
*
bs3  
/functions_comparison.yaml
SOhttps://github.com/apache/arrow/blob/main/format/substrait/extension_types.yamlequal:any1_any1	
u64
	u32

"
 "
("i
i64
f64
str
const
struct
a
b
group
key7
:
Z
b
:

:
b
*
br  rd   4)
pyarrow.substrait	substraitr9  r  r  r   BoundExpressionsfrom_substraitr   rB  )rX   psrJ  	filteringrI  s        r   test_scanner_from_substraitr(    s    -&
WJ
CI __##22:>""11)<   hj  %#s!44449  -+,-s   A7 7 BBr  r   )r   r"   r   (	  r   r0   ry  r  r  r  sysr  r  r  r  shutilr   urllib.parser   numpyrr  r9  r  r   rD   rW  rX  r  pyarrow.csvr  r   rh   pyarrow.jsonpyarrow.libr   pyarrow.tests.utilr   r   r   r	   r
   r   r8   r  rX   r   pyarrow.parquetr   rq   mark
pytestmarkr   r@   rN   r[   fixturers   r   r   r   r  r)  r.  rK  rY  r_  rg  r{  r  r  r  r  r  r  r  parametrizerd   tupler  r  r  r  r  r"  r<  rH  r   rZ  ra  s3rt  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r
  r  r  r  r  r  r   r-  r5  rC  rL  r_  rf  rl  rn  rw  r  r  r  r  r  r  r  r  r  r3  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r(  r-  rh  r9  r<  rE  rS  rW  r\  rb  re  rg  rl  rq  r{  r~  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  skipifplatformr  r  r  r  r$  r&  r(  r0  r3  r7  r:  r>  rC  rK  rQ  rT  r^  re  rh  ro  rt  r{  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r   r)  r/  r2  r?  rG  rN  rU  rX  r[  r]  rb  rm  rn  rq  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r(  r    r   r   <module>r:     s  $   	    
              ,, ,   [[  
? ?K +%  D      F  @ h0  0f   	
 	H+	+\ DG DGN), ?9 ?9D)GX&( " " -I -I` % %$ - -, 5 5 O OA!HF$ F8
8
 %8
 #(8
8
v#"6 & &R $ $" 6 6r$TNJ2 ,OOH-$$/  t}5E"  6E"P 4 4$ 5-  5-pH2H, *D *DZ  " 7 7@ 	%  	% 9N 9Nx  : ) )" 
:  
: %0 %0P  "5Jp '  'T 7 7 7  7 ) ), &. &.R $ $6 ' '> 	 	  3  3F     F  @ACIC I< +eT]; @AC@C < @@  @ACa,Ca,H  @ACR,CR,jJ? (  (2N
+;
=O"6J H H(*8,>@" I I
 I I L L $ $$ < <& J J& 7 7&    , - - = =*"?(DJ" 6 7&" !# !#H ? ?, , , N N.A +v)>?5$-8+eT];)i  o&	?# !< , 62 < 9 @ 62r ; ;< D D& 
:  
: :  : ,  ,0 ?=  ?=D / / 9 92 = =6 F F 	 	 9 9(4?%DM2@ 6) 6)r  "$, #M #ML ? ?,*    "    $  )    (H*: A0    (    (    ,D2@0G@  ) 
,/ ,/^ , !  ! CLLG+D  F!F  !( !  !2 . .* 	  	$3 4  4$ 
;  
; 3  3N @  @  G G* D  D 	=  	==0 :?  :?z B  B&( ?C3&  P  PF *  *>)")"=& H  HVF" -# -#`;	O !< !<H  <  <F $ $8 A9 A9H #  #0 #9  #9L   ,<^!H 6@4 0 ('  ('V 4  *  ,  ,F 4  4	1 1 1 ,   , ^ $ I
  I
X " "$  <  < A A6  6  :  . 
 
> %$ K>K>\ %$ *1*1Z$
2 #56 76"'J = =.R:R@ %$ -C-C`FR'&5iw  	B"  	B
  	B
  	Bs[   AT 3AT( 8AT7 ?AU TAT%T$AT%T(AT4T3AT4T7AUUAUUAUUAU