
    <>i'0                       S SK r S SKrS SKJrJr  S SKJr  S SKrS SKrS SKrS SK	r	S SK
r
S SKrS SKrS SKrS SKrS SKrS SKrS SKrS SKrS SKrS SKrS SKrS SKrS SKJrJrJrJrJrJrJrJ r J!r!J"r"  S SK#J$r$  S r%S r&S r'S4S	 jr(S
 r)S r*S r+ " S S5      r,S r-S r.S r/S r0S r1 " S S\ Rd                  5      r3 " S S\35      r4 " S S\45      r5 " S S\45      r6 " S S\35      r7 " S S\75      r8\Rr                  R"                   " S  S!\75      5       r: " S" S#5      r; " S$ S%\;\Rx                  5      r= " S& S'\;\Rx                  5      r>S( r?S) r@S* rAS+ rBS, rC\Rr                  R                  S-S. S/ S0 S1 45      S2 5       rES3 rFg)5    N)datedatetime)Decimal)
open_csvread_csvReadOptionsParseOptionsConvertOptionsISO8601	write_csvWriteOptions	CSVWriter
InvalidRow)utilc               #   t   #    [         R                  n U  S h  vN   U  H  nU  H	  nX-   v   M     M     g  N7fN)stringascii_lowercase)lettersfirstseconds      U/var/www/html/land-tabula/venv/lib/python3.13/site-packages/pyarrow/tests/test_csv.pygenerate_col_namesr   /   s:     $$GF.    s   868c              #   H   #    [        SX!-  U5       H  nXX1-    v   M     g 7fNr   rangearrnum_colsnum_rowsis       r   
split_rowsr#   8   s(     1h)84AL!! 5s    "c              #   D   #    [        SU5       H  nXS U2   v   M     g 7fr   r   r   s       r   split_columnsr%   >   s%     1h+X+  s    c           	      ,   [         R                  " S5      n[        X-  5       Vs/ s H  oTR                  SS5      PM     nn[        R
                  " 5       n[        [        R                  " [        5       U 5      5      nU(       a1  UR                  SR                  U5      5        UR                  U5        [        X`U5       HB  n	UR                  SR                  [        [        U	5      5      5        UR                  U5        MD     UR                  5       R!                  5       n[#        X`U5       V	s/ s H+  n	[$        R&                  " U	[$        R(                  " 5       S9PM-     n
n	[$        R*                  R-                  X5      nX{4$ s  snf s  sn	f )N*   r     ,)type)randomRandomr   randintioStringIOlist	itertoolsislicer   writejoinr#   mapstrgetvalueencoder%   paarrayint64Tablefrom_arrays)r    r!   linesepwrite_namesrnd_r   csv	col_namesrowcolumnsexpecteds               r   make_random_csvrG   D   s1   
--
C).x/B)C
D)CA;;q$)CC
D
++-CY%%&8&:HEFI		#((9%&		'#2		#((3sC=)*		' 3 ,,.


!C'x@B@3 xx"((*-@  Bxx##G7H= EBs   F62Fc                     [         R                  " 5       nUR                  SR                  U 5      5        UR                  S5        UR	                  5       R                  5       $ )Nr)   
)r.   r/   r3   r4   r7   r8   )column_namesrB   s     r   make_empty_csvrK   V   sB    
++-CIIchh|$%IIdO<<>  ""    c                    U " 5       nUR                  5        HL  u  p4[        X#5      US   :X  d
   SU-   5       eU H%  n[        X#U5        [        X#5      U:X  a  M    S5       e   MN     [        R                  " [
        5         SUl        SSS5        UR                  5        VVs0 s H
  u  p4X4S   _M     nnnU " S0 UD6nUR                  5        H  u  p7[        X#5      U:X  a  M   e   g! , (       d  f       Nh= fs  snnf )z<
Check setting and getting attributes of an *Options class.
r   zincorrect default value for zfailed setting valueTN    )itemsgetattrsetattrpytestraisesAttributeErrorzzz_non_existent)clsattr_valuesoptsnamevaluesvnon_defaultsvalues           r   check_options_classr_   ]   s     5D#))+t"fQi/ 	2*T1	2/AD"4&!+C-CC+  , 
~	& $ 
' 9D8I8I8KL8KD)O8KLLD#))+t"e+++ , 
'	& Ms   C,%C=,
C:c                     U " S0 UD6nUR                  UR                  UUR                  S95      nUR                  5        H  u  pV[	        XE5      U:X  a  M   e   g )N)protocolrO   )loadsdumpsHIGHEST_PROTOCOLrP   rQ   )rW   picklerrX   rY   new_optsrZ   r^   s          r   check_options_class_picklingrg   u   sb    D}}W]]44;4L4L + N OH"((*x&%/// +rL   c                   ,    \ rS rSrS rS rS rS rSrg)InvalidRowHandler}   c                     Xl         / U l        g r   resultrows)selfrm   s     r   __init__InvalidRowHandler.__init__~   s    	rL   c                 P    U R                   R                  U5        U R                  $ r   )rn   appendrm   )ro   rD   s     r   __call__InvalidRowHandler.__call__   s    		{{rL   c                 b    [        U[        5      =(       a    UR                  U R                  :H  $ r   
isinstanceri   rm   ro   others     r   __eq__InvalidRowHandler.__eq__   s'    5"34 ,+	-rL   c                 l    [        U[        5      (       + =(       d    UR                  U R                  :g  $ r   rw   ry   s     r   __ne__InvalidRowHandler.__ne__   s*    u&788 ,+	-rL   rl   N)	__name__
__module____qualname____firstlineno__rp   rt   r{   r~   __static_attributes__rO   rL   r   ri   ri   }   s    --rL   ri   c                      Sn Sn[        SS5       Vs/ s H  o"PM     nn[        [        X0U5      5      nU/ SQ/ SQ/:X  d   e[        [        X0U5      5      nUSS/SS/S	S
/SS/SS//:X  d   eg s  snf )N      rN      )rN   r         r   )         	   
   r   r   r   r   r   r   r   )r   r0   r#   r%   )r    r!   xr   rn   rE   s         r   #test_split_rows_and_columns_utilityr      s    HHArl
#l1lC
#
3(34D    =9:G	
A	
A	
A	
A	
B    $s   A0c                    [         nU" 5       n[        USS/SS// SS//SS/SS/SS	/S
9  [        XSSSS/SSS	S9  UR                  S:  d   eSUl        UR                  S:X  d   eU" SS9nUR                  S:X  d   eUR	                  5         Sn[
        R                  " [        R                  US9   U" 5       nSUl        UR	                  5         S S S 5        Sn[
        R                  " [        R                  US9   U" 5       nSUl	        UR	                  5         S S S 5        Sn[
        R                  " [        R                  US9   U" 5       nSUl
        UR	                  5         S S S 5        Sn[
        R                  " [        R                  US9   U" 5       nSUl        SUl        UR	                  5         S S S 5        g ! , (       d  f       N= f! , (       d  f       N= f! , (       d  f       N= f! , (       d  f       g = f)NTFr   r   abcdutf8utf16   )use_threads	skip_rowsrJ   autogenerate_column_namesencodingskip_rows_after_names)re   r   r   rJ   r   r   r   90  i  
block_sizez-ReadOptions: block_size must be at least 1: 0matchz-ReadOptions: skip_rows cannot be negative: -1z9ReadOptions: skip_rows_after_names cannot be negative: -1zTReadOptions: autogenerate_column_names cannot be true when column_names are providedab)r   r_   rg   r   validaterS   rT   r9   ArrowInvalidr   r   r   rJ   )pickle_modulerW   rY   r   s       r   test_read_optionsr      s   
C5D$#$a&&(4,%738$-"('!2/0"g7 !-1+,/3Tl;@*179; ??QDO??e###$D??d"""MMO;E	re	4u 
5
 <E	re	4u 
5
 HE	re	4u%'" 
5
)E	re	4u)-&&	 
5	4' 
5	4 
5	4 
5	4 
5	4s0   4G GG)&G:
G
G&)
G7:
Hc                 >   [         n[        S5      n[        USS/SS// SQSS/SS/SS/S U/S9  [        XSSSSSSUS	9	  U" 5       R	                  5         U" 5       nS
Ul        UR	                  5         Sn[        R                  " [        R                  US9   U" 5       nSUl        UR	                  5         S S S 5        [        R                  " [        R                  US9   U" 5       nSUl        UR	                  5         S S S 5        Sn[        R                  " [        R                  US9   U" 5       nSUl
        UR	                  5         S S S 5        [        R                  " [        R                  US9   U" 5       nSUl
        UR	                  5         S S S 5        Sn[        R                  " [        R                  US9   U" 5       nSUl        UR	                  5         S S S 5        [        R                  " [        R                  US9   U" 5       nSUl        UR	                  5         S S S 5        g ! , (       d  f       GN= f! , (       d  f       GNN= f! , (       d  f       GN= f! , (       d  f       N= f! , (       d  f       N= f! , (       d  f       g = f)Nskipr)   r   Fy)"zFT)	delimiterescape_char
quote_chardouble_quotenewlines_in_valuesignore_empty_linesinvalid_row_handler)re   r   r   r   r   r   r   r   	z,ParseOptions: delimiter cannot be \\r or \\nr   rI   z-ParseOptions: quote_char cannot be \\r or \\nz.ParseOptions: escape_char cannot be \\r or \\n)r	   ri   r_   rg   r   r   rS   rT   r9   r   r   r   )r   rW   skip_handlerrY   r   s        r   test_parse_optionsr      s   
C$V,LSz%*CL#4&*E],14=,0%=-1<,@B !+.-0,1.348495AC ENN5DDNMMO>E	re	4u 
5
 
re	4u 
5
 @E	re	4u 
5
 
re	4u 
5
 AE	re	4u 
5
 
re	4u 
5	47 
5	4
 
5	4 
5	4
 
5	4 
5	4
 
5	4sH   H6"I.I8I,I=J6
I
I
I),
I:=
J
Jc                 	   [         nU" 5       n[        USS/SS/SS/SS// SS//SS/SS// [        S//S9	  [        XSSSSSS/SS[        S/S	9
  [        R
                  " [        5         S
Ul        S S S 5        UR                  S:  d   eSUl        UR                  S:X  d   eUR                  0 :X  d   e[        R                  " 5       [        R                  " 5       S.Ul	        UR                  [        R                  " 5       [        R                  " 5       S.:X  d   eSSS.Ul	        UR                  [        R                  " 5       [        R                  " 5       S.:X  d   e[        R                  " S[        R                  " 5       4S[        R                   " 5       4/5      nX2l	        UR                  [        R                  " 5       [        R                   " 5       S.:X  d   eS[        R"                  " 5       4/Ul	        UR                  S[        R"                  " 5       0:X  d   e[        R
                  " [$        SS9   SS 0Ul	        S S S 5        [        R
                  " [$        5         SUl	        S S S 5        ['        UR(                  [*        5      (       d   eSUR(                  ;   d   eSUR(                  ;   d   eSS/Ul        UR(                  SS/:X  d   e['        UR,                  [*        5      (       d   eSS/Ul        UR,                  SS/:X  d   e['        UR.                  [*        5      (       d   eSS/Ul        UR.                  SS/:X  d   eUR0                  / :X  d   e[        /Ul        UR0                  [        /:X  d   eU" S[        R                  " 5       0SS/SS/SS /S![        S"/S#9nUR                  S[        R                  " 5       0:X  d   eUR(                  SS/:X  d   eUR.                  SS /:X  d   eUR,                  SS/:X  d   eUR                  S!:X  d   eUR0                  [        S"/:X  d   eg ! , (       d  f       GN= f! , (       d  f       GN= f! , (       d  f       GN = f)$NTF.r)   defabcz%y-%m)
check_utf8strings_can_be_nullquoted_strings_can_be_nulldecimal_pointinclude_columnsinclude_missing_columnsauto_dict_encodetimestamp_parsers)	re   r   r   r   r   r   r   r   r   z..r   韆 )r   cint16null)r\   wr   r   r   r   zDataType expectedr    N/AxxxyyyNnnTttFffi  z%Y-%m-%d)column_typesnull_valuestrue_valuesfalse_valuesauto_dict_max_cardinalityr   )r
   r_   r   rg   rS   rT   
ValueErrorr   r   r   r9   r   float32r   schemaint32r   binary	TypeErrorrx   r   r0   r   r   r   )r   rW   rY   r   s       r   test_convert_optionsr     s   
C5Du"DM$(%=CjeU^,!&124 ! #( %"G,	. 
z	"! 
# ))A---%*D"))U222""" hhjrzz|<Dbhhjrzz| DDDD%F3Dbhhjrwwy AAAAYYbhhj)C+=>?Fbhhjryy{ CCCCryy{+,Dbiik 2222	y(;	< $K 
=	y	! 
" d&&----!!!!!D$$$$$u~Du~---d&&----u~Du~---d''....D...!!R'''%YD!!gY...S"''),;S$K $K3"):!68D bggi 0000T{***d+++T{***))S000!!gz%::::k 
#	"* 
=	<	!	!s$   *Q
Q"3Q4
Q"
Q14
Rc                     [         n U " 5       n[        U SS// SQ/ SQS9  UR                  S:  d   eSUl        UR                  S:X  d   eU " SS	9nUR                  S:X  d   eUR                  5         S
n[        R
                  " [        R                  US9   U " 5       nSUl        UR                  5         S S S 5        g ! , (       d  f       g = f)NTF)r)   r   |)needednone	all_valid)include_headerr   quoting_styler   r   i&  )
batch_sizez.WriteOptions: batch_size must be at least 1: 0r   )r   r_   r   r   rS   rT   r9   r   )rW   rY   r   s      r   test_write_optionsr   f  s    
C5DT5M5E57 ??QDO??e###$D??d"""MMO<E	re	4u 
5	4	4s   C  
Cc                       \ rS rSrSr\R                  S 5       r\\R                  S 5       5       r	\
S 5       rS rS r\R                  R                   S 5       rS	 rS
 rSrg)BaseTestCSVi~  zDCommon tests which are shared by streaming and non streaming readersc                     [         e)zz
:param b: bytes to be parsed
:param kwargs: arguments passed on to open the csv file
:return: b parsed as a single Table
NotImplementedErrorro   r   kwargss      r   
read_bytesBaseTestCSV.read_bytes  s
     "!rL   c                     [         e)z#Whether this test is multi-threadedr   ro   s    r   r   BaseTestCSV.use_threads  s
     "!rL   c                 ^    U R                   [        U5      :X  d   eU R                  U:X  d   eg r   )num_columnslenrJ   )tablenamess     r   check_namesBaseTestCSV.check_names  s.      CJ...!!U***rL   c                    Sn[        5       nSUl        U R                  XS9nU R                  USS/5        UR	                  5       SS/SS	/S
.:X  d   eSUl        U R                  XS9nU R                  USS	/5        UR	                  5       / / S.:X  d   eSUl        [
        R                  " [        R                  5         U R                  XS9nS S S 5        SnSUl        U R                  XS9nU R                  USS/5        UR	                  5       S/S	/S.:X  d   eSUl        SS/Ul	        U R                  XS9nU R                  USS/5        UR	                  5       / / S.:X  d   eg ! , (       d  f       N= f)N   ab,cd
ef,gh
ij,kl
mn,op
rN   read_optionsefghijmnklop)r  r  r   )r  r  r      abcd
,,,,,
ij,kl
mn,op
r   )r  r  )
r   r   r   r   	to_pydictrS   rT   r9   r   rJ   ro   rn   rY   r   s       r   test_header_skip_rows!BaseTestCSV.test_header_skip_rows  s   .}8t- ,,%
 
 	
 

 8t- %
 
 	
 

 ]]2??+OODO<E ,
 .8t- &&%
 
 	
 
 !4L8t- %
 
 	
 
' ,+s   E..
E<c                    Sn[        5       nSUl        U R                  XS9nU R                  USS/5        UR	                  5       SS/SS	/S
.:X  d   eSUl        U R                  XS9nU R                  USS/5        UR	                  5       / / S
.:X  d   eSUl        U R                  XS9nU R                  USS/5        UR	                  5       / / S
.:X  d   eSnSUl        SS/Ul        U R                  XS9nU R                  USS/5        UR	                  5       SS/SS	/S.:X  d   e[        5       nSnSUl        [        5       nSUl        U R                  XUS9nU R                  USS/5        UR	                  5       S/S	/S
.:X  d   eSUl        SUl        U R                  XUS9nU R                  USS/5        UR	                  5       S/S	/S
.:X  d   e[        5       n[        SSS9u  pSUl        [        U5      S-  Ul        U R                  XS9nUR                  UR                  :X  d   eUR                  S:X  d   eUR	                  5       nUR	                  5       R                  5        H  u  pxUSS  Xg   :X  a  M   e   U R                  XUS9nUR                  UR                  :X  d   eUR                  S:X  d   eUR	                  5       nUR	                  5       R                  5        H  u  pxUSS  Xg   :X  a  M   e   [        SSSS9u  p[        5       nSUl        S Ul        UR                  R                  Ul        U R                  XUS9nUR                  UR                  :X  d   eUR                  UR                  UR                  -
  UR                  -
  :X  d   eUR	                  5       nUR	                  5       R                  5        H(  u  pxXR                  UR                  -   S  Xg   :X  a  M(   e   g )!Nr   rN   r   r   r   r  r  r  r  r   r   r   r   r  r   f0f1r  r  s#   ab,cd
"e
f","g

h"
"ij","k
l"
mn,opTr   parse_options   r   r(   r    r!   i  r   d      F)r    r!   r?   %   )   )r   r   r   r   r  rJ   r	   r   r   rG   r   r   r!   rP   r   r   )	ro   rn   rY   r   
parse_optsrF   
table_dictrZ   r[   s	            r   test_skip_rows_after_names&BaseTestCSV.test_skip_rows_after_names  s{   .}%&"8t- ,,%
 
 	
 
 &'"8t- %
 
 	
 
 &'"8t- %
 
 	
 
 .%&"!4L8t- ,,%
 
 	
 
 } =%&"!^
(,
%.8   :t- &&%
 
 	
 
 &'".8   :t- &&%
 
 	
 
 } )!dC%("d)b.8||x...~~$$$__&
$..0668LD#$<:#3333 9 .8   :||x...~~$$$__&
$..0668LD#$<:#3333 9 )!c5:<}%'"$OO11.8   :||x...!!DNN2**++ 	, + __&
$..0668LD>>D,F,FFGH$% & % 9rL   c                   ^  U 4S jn[        SSSS9u  p#[        5       n[        U5      S-  Ul        [	        5       nS[
        R                  " 5       0Ul        US-   nU" S	S
5      n[        R                  " [
        R                  US9   T R                  UUUS9  S S S 5        US-   nU" SS
U5      n	[        R                  " [
        R                  U	S9   T R                  UUUS9  S S S 5        Sn
X*-   nU" SS
U
SS R                  S5      5      n[        R                  " [
        R                  US9   T R                  UUUS9  S S S 5        SUl        [        R                  " [
        R                  US9   T R                  UUUS9  S S S 5        [        R                  " [
        R                  U	S9   T R                  UUUS9  S S S 5        [        R                  " [
        R                  US9   T R                  UUUS9  S S S 5        SUl        [        SSSS9u  p#/ SQUl        US-   nU" S	S5      n[        R                  " [
        R                  US9   T R                  UUUS9  S S S 5        X*-   nU" SSU
SS R                  S5      5      n[        R                  " [
        R                  US9   T R                  UUUS9  S S S 5        US-   nU" SS5      n	U	R                  [        U5      5      n	[        R                  " [
        R                  U	S9   T R                  UUUS9  S S S 5        SUl        [        R                  " [
        R                  US9   T R                  UUUS9  S S S 5        [        R                  " [
        R                  U	S9   T R                  UUUS9  S S S 5        g ! , (       d  f       GN$= f! , (       d  f       GN= f! , (       d  f       GN= f! , (       d  f       GNk= f! , (       d  f       GN@= f! , (       d  f       GN= f! , (       d  f       GN= f! , (       d  f       GNt= f! , (       d  f       GN!= f! , (       d  f       N= f! , (       d  f       g = f)Nc                 ^   > TR                   (       a  SnOSU S3nU R                  " U/UQ76 $ )Nr   zRow #z: )r   format)
msg_formatrD   argsrow_inforo   s       r   
format_msg@BaseTestCSV.test_row_number_offset_in_errors.<locals>.format_msg.  s4    "3%r?$$X555rL   r   r  T)r?   r   r   s   1,2
z{}Expected 4 columns, got 2f   r   r   convert_optionss	   a,b,c,d
zDIn CSV column #0: {}CSV conversion error to int32: invalid value 'a's  this is a long rowthis is a long rowthis is a long rowthis is a long rowthis is a long rowthis is a long rowthis is a long rowthis is a long rowthis is a long rowthis is a long rowthis is a long rowthis is a long rowthis is a long rowthis is a long rowthis is a long row,3
z#{}Expected 4 columns, got 2: {} ...r   `   zutf-8/   Fr   r   r   de      )rG   r   r   r   r
   r9   r   r   rS   rT   r   r   decoder   rJ   r  r   )ro   r#  rB   rA   r   r'  csv_bad_columnsmessage_columnscsv_bad_typemessage_valuelong_rowcsv_bad_columns_longmessage_longs   `            r    test_row_number_offset_in_errors,BaseTestCSV.test_row_number_offset_in_errors+  s   	6 !CT:"}"%c(Q,(*(+RXXZ'8$ 
*$%BCH]]2??/BOOO)5,;  = C
 ^+"? ]]2??-@OOL)5,;  = A
 <"~!"G"*1R."7"7"@B]]2??,?OO0)5,;  = @ .0*]]2??/BOOO)5,;  = C
 ]]2??-@OOL)5,;  = A
 ]]2??,?OO0)5,;  = @
 ./* !CU;$8!
*$%BCH]]2??/BOOO)5,;  = C
  #~!"G"*1R."7"7"@B]]2??,?OO0)5,;  = @
 ^+"? &,,SX6]]2??-@OOL)5,;  = A "$]]2??/BOOO)5,;  = C
 ]]2??-@OOL)5,;  = A@a CB A@ @? CB
 A@
 @? CB @? A@ CB
 A@s   	N8O
4O8O.5P 2PP$7P6Q QQ+8
O

O
O+.
O= 
P
P!$
P36
Q
Q
Q(+
Q9c                 n  ^  Sn[        5       n[        R                  " [        SS9   T R	                  X#S9  S S S 5        [        S5      Ul        T R	                  X#S9nUR                  5       SS/SS	/S
.:X  d   eU 4S jn[        SSU" S5      S5      [        SSU" S5      S5      /nUR                  R                  U:X  d   e[        S5      Ul        [        R                  " [        SS9   T R	                  X#S9  S S S 5        [        SSU" S5      S5      /nUR                  R                  U:X  d   e[        S5      Ul        UR                  UR                  U5      5      nT R	                  X#S9nUR                  5       SS/SS	/S
.:X  d   eg ! , (       d  f       GNd= f! , (       d  f       N= f)Ns   a,b
c
d,e
f,g,h
i,j
zExpected 2 columns, got 1: cr   r  r   r+  r"   ejr   c                 .   > TR                   (       a  S $ U $ r   )r   )r   ro   s    r   row_num5BaseTestCSV.test_invalid_row_handler.<locals>.row_num  s    ++422rL   r   rN   r   r   r   zf,g,herror)r	   rS   rT   r   r   ri   r   r  r   rn   rb   rc   )ro   r   rn   r  r   r=  expected_rowss   `      r   test_invalid_row_handler$BaseTestCSV.test_invalid_row_handler  s   +!^
]]46 OODO;6 *;6)B
&? ss%
 
 	
 

	3 q!WQZ-q!WQZ1
 --22mCCC *;7)C
&]]46 OODO;6 $Aq'!*c:;--22mCCC *;6)B
&"(()<)<Z)HI
? ss%
 
 	
 
G6 6.6 6s   F.F&
F#&
F4c           	      j   SnSS/SS/SS/S.n[        S	S
5       H7  nU R                  U[        SS9[        US9S9R	                  5       nXB:X  a  M7   e   [        S	S5       H<  n[
        R                  " [        SS9   U R                  U[        US9S9  S S S 5        M>     g ! , (       d  f       MP  = f)Ns   a,b,c
d,e,"f
"
g,h,i
r+  gr:  hzf
r"   r   r   r   r      T)r   r   r  r   r   z#cell values spanning multiple linesr   r   )r   r   r	   r   r  rS   rT   r   )ro   rn   rF   r   r+  s        r   test_chunker_out_of_sync$BaseTestCSV.test_chunker_out_of_sync  s     0ss

  2,JLD$I(J?   AAJ  = = '  2,Jz%JL{j'I   KL L 'L Ls   >B##
B2	rO   N)r   r   r   r   __doc__r   abstractmethodr   propertyr   staticmethodr   r
  r  rS   marknumpyr6  rA  rI  r   rO   rL   r   r   r   ~  s    N" " "  " + ++
Zg&R [[f= f=P)
VKrL   r   c                   L   \ rS rSrSS.S jrS rS rS rS rS	 r	S
 r
S rS rS rS rS rS rS rS rS rS rS rS rS rS rS rS rS rS rS rS rS rS  r S! r!S" r"S# r#\$RJ                  RL                  S$ 5       r'\$RJ                  RL                  S% 5       r(S&r)g')(BaseCSVTableReadi  T)validate_fullc                    [        U R                  [        5      (       d   eUR                  S[	        5       5      nU R                  Ul        [        U/UQ70 UD6nUR                  US9  U$ )a  
Reads the CSV file into memory using pyarrow's read_csv
csv The CSV bytes
args Positional arguments to be forwarded to pyarrow's read_csv
validate_full Whether or not to fully validate the resulting table
kwargs Keyword arguments to be forwarded to pyarrow's read_csv
r   full)rx   r   bool
setdefaultr   r   r   )ro   rB   rS  r!  r   r   r   s          r   r   BaseCSVTableRead.read_csv  sf     $**D1111((G#'#3#3 .t.v.M*rL   c                 P    U R                   " [        R                  " U5      40 UD6$ r   )r   r9   	py_bufferr   s      r   r   BaseCSVTableRead.read_bytes      }}R\\!_777rL   c                 b   SnS/S/S.n[         R                  " U5      nU R                  U5      nUR                  5       U:X  d   e[         R                  " UR                  5       5      n[        R                  " [        5         U R                  U5        S S S 5        g ! , (       d  f       g = f)Ns   a,b
1,2
rN   r   r   )	r.   BytesIOr   r  r/   r.  rS   rT   r   )ro   dataexpected_databior   sios         r   test_file_object!BaseCSVTableRead.test_file_object  s~     c,jjc" M111kk$++-(]]9%MM# &%%s   B  
B.c                     SnU R                  U5      n[        U[        R                  5      (       d   eU R	                  U/ SQ5        UR
                  S:X  d   eg )N   abc,def,gh
)r   r   r  r   )r   rx   r9   r<   r   r!   ro   rn   r   s      r   test_headerBaseCSVTableRead.test_header  sO    %%**** 45~~"""rL   c                 d    SnS/S/S.nU R                  U5      nUR                  5       U:X  d   eg )Ns   ﻿a,b
1,2
rN   r   r   r   r  )ro   rn   ra  r   s       r   test_bomBaseCSVTableRead.test_bom  s8    ( c,% M111rL   c                     / SQnS Ho  nSU4 Hd  nUR                  U5      U-   nU R                  U5      n[        UR                  5       5      S:X  d   eUR	                  5       / SQ/ SQS.:X  a  Md   e   Mq     g )N)s   a,bs   1,2s   3,4s   56,78)   
      
rL   rN   )rN   r   8   )r   r   N   r   )r4   r   r   
to_batchesr  )ro   rn   line_endingfile_endingr`  r   s         r   test_one_chunkBaseCSVTableRead.test_one_chunk  s     22K #[1"''-;-5++-.!333(##-   	  2 3rL   c                    Sn[        5       nSS/Ul        U R                  XS9nU R                  USS/5        UR	                  5       / SQ/ SQS.:X  d   eSUl        U R                  XS9nU R                  USS/5        UR	                  5       S	/S
/S.:X  d   eSUl        U R                  XS9nU R                  USS/5        UR	                  5       / / S.:X  d   eSUl        [        R                  " [        R                  5         U R                  XS9nS S S 5        SUl        / SQUl        [        R                  " [        R                  SS9   U R                  XS9nS S S 5        SnSUl        SS/Ul        U R                  XS9nU R                  USS/5        UR	                  5       SS	/SS
/S.:X  d   eg ! , (       d  f       N= f! , (       d  f       Nr= f)Nr   r   r   r   r   r  r  r  r   r  r  r  r   r   r   r  r  r   r   r   )r   r   r   Expected 3 columns, got 2r   r  r   r  r  )
r   rJ   r   r   r  r   rS   rT   r9   r   r	  s       r   test_header_column_names)BaseCSVTableRead.test_header_column_names  s   .} #J8c
+ ))%
 
 	
 

 8c
+ %
 
 	
 

 8c
+ %
 
 	
 

 ]]2??+OODO<E ,
 +]]2??!<>OODO<E>
 . #J8c
+ %
 
 	
 
# ,+> >s   	F?G?
G
Gc                    Sn[        5       nSUl        U R                  XS9nU R                  USS/5        UR	                  5       / SQ/ SQS.:X  d   eS	Ul        U R                  XS9nU R                  USS/5        UR	                  5       S
/S/S.:X  d   eSUl        [        R                  " [        R                  5         U R                  XS9nS S S 5        g ! , (       d  f       g = f)Nr   Tr   r  r  r{  r|  r  r   r  r  r   )
r   r   r   r   r  r   rS   rT   r9   r   r	  s       r   %test_header_autogenerate_column_names6BaseCSVTableRead.test_header_autogenerate_column_namesF  s    .})-&8t- **%
 
 	
 

 8t- &&%
 
 	
 
 ]]2??+OODO<E ,++s   C
C+c                 L   Sn[        5       nS/Ul        U R                  XS9nU R                  US/5        UR	                  5       S/ SQ0:X  d   eSS/Ul        U R                  XS9n[
        R                  " S[
        R                  " 5       4S[
        R                  " 5       4/5      nUR                  U:X  d   eUR	                  5       / SQ/ SQS.:X  d   e/ SQUl        [        R                  " [        S	S
9   U R                  XS9  S S S 5        g ! , (       d  f       g = f)Nr   r   r'  r  r  r  r   r  r  r  )r   r   xxr   yyz9Column 'xx' in include_columns does not exist in CSV filer   )r
   r   r   r   r  r9   r   r   rS   rT   KeyError)ro   rn   r'  r   r   s        r   test_include_columns%BaseCSVTableRead.test_include_columns_  s   .(*+/&'F' $%
 
 	
 

 ,0,'FT299;/!299;/1 2||v%%% $$%
 
 	
 
 +=']]8">? OODOB? ? ?s   <D
D#c                    Sn[        5       n[        5       n/ SQUl        SUl        U R	                  XUS9n[
        R                  " S[
        R                  " 5       4S[
        R                  " 5       4S[
        R                  " 5       4/5      nUR                  U:X  d   eUR                  5       / SQ/ S	Q/ SQS.:X  d   eSS/Ul
        SS
/Ul        U R	                  XUS9n[
        R                  " S[
        R                  " 5       4S
[
        R                  " 5       4/5      nUR                  U:X  d   eUR                  5       / SQ/ SQS.:X  d   e[
        R                  " 5       [
        R                  " 5       S.Ul        U R	                  XUS9n[
        R                  " S[
        R                  " 5       4S
[
        R                  " 5       4/5      nUR                  U:X  d   eUR                  5       / SQ/ SQS.:X  d   eg )Nr   r  Tr&  r  r   r  NNNr  r   r|  )NNNN)r  r   )s   cds   ghs   kls   op)r   r
   r   r   r   r9   r   r   r   r  rJ   r   r   r   ro   rn   r   r'  r   r   s         r   test_include_missing_columns-BaseCSVTableRead.test_include_missing_columns|  s   ."}(**<'26/0?   AT2779-!299;/!2779-/ 0 ||v%%% $$$%
 
 	
 
 &*4L!+/,'0?   AT299;/!2779-/ 0||v%%% **%
 
 	
 
 /1iik.0hhj(:$0?   AT299;/!288:.0 1||v%%% .*%
 
 	
 
rL   c                 >   SnU R                  U5      n[        R                  " S[        R                  " 5       4S[        R                  " 5       4S[        R                  " 5       4/5      nUR                  U:X  d   eUR	                  5       SS/SS/S	S
/S.:X  d   eg )Ns   a,b,c
1,2,3
4,5,6
r   r   r   rN   r   r   r   r   r   rF  )r   r9   r   r;   r  ro   rn   r   r   s       r   test_simple_ints!BaseCSVTableRead.test_simple_ints  s    '%S"((*- "((*- "((*-/ 0 ||v%%% QQQ%
 
 	
 
rL   c                 p   SnU R                  U5      n[        R                  " S[        R                  " 5       4S[        R                  " 5       4S[        R
                  " 5       4S[        R                  " 5       4/5      nUR                  U:X  d   eUR                  5       SS/SS	/S
S/SS/S.:X  d   eg )Ns    a,b,c,d
1,2,3,0
4.0,-5,foo,True
r   r   r   r+        ?g      @r   3fooFTr*  )r   r9   r   float64r;   r   bool_r  r  s       r   test_simple_varied#BaseCSVTableRead.test_simple_varied  s    5%S"**,/ "((*- "))+. "((*-/ 0 ||v%%% sRu	%
 
 	
 
rL   c           
         SnU R                  U5      n[        R                  " S[        R                  " 5       4S[        R                  " 5       4S[        R
                  " 5       4S[        R                  " 5       4S[        R                  " 5       4S[        R                  " 5       4/5      nUR                  U:X  d   eUR                  5       / SQ/ S	Q/ S
Q/ SQ/ SQ/ SQS.:X  d   eg )NsC   a,b,c,d,e,f
1,2,,,3,N/A
nan,-5,foo,,nan,TRUE
4.5,#N/A,nan,,,false
r   r   r   r+  r:  f)r  Ng      @)r   r  N)r   r  nanr  )   3s   nan   )NTF)r   r   r   r+  r:  r  )
r   r9   r   r  r;   r   r   r   r  r  r  s       r   test_simple_nulls"BaseCSVTableRead.test_simple_nulls  s    . %S"**,/ "((*- "))+. "''), "))+. "((*-/ 0 ||v%%% !##($%
 
 	
 
rL   c                 .   [        SS9nSnU R                  X!S9n[        R                  " S[        R                  " 5       4S[        R
                  " 5       4/5      nUR                  U:X  d   eUR                  5       / SQ/ SQS	.:X  d   e[        S
S9nU R                  X!US9n[        R                  " S[        R
                  " 5       4S[        R                  " 5       4/5      nUR                  U:X  d   eUR                  5       / SQ/ SQS	.:X  d   eg )N;r   s   a;b
1.25;2,5
NA;-3
-4;NAr9  r   r   )g      ?Ng      )z2,5z-3NAr   r)   )r   )r  r'  )z1.25r  z-4)g      @g      N)r	   r   r9   r   r  r   r  r
   )ro   r  rn   r   r   r'  s         r   test_decimal_point#BaseCSVTableRead.test_decimal_point  s   $s3-BS"**,/ "))+.0 1||v%%% #$%
 
 	
 

 )s;0?   AS"))+. "**,/1 2||v%%% %"%
 
 	
 
rL   c                    SnU R                  U5      n[        R                  " S[        R                  " 5       4S[        R                  " S5      4S[        R                  " S5      4/5      nUR                  U:X  d   eUR                  5       SS/[        SS	S	5      [        SS
SS	5      /[        SS	S	SSSS5      [        SS
SS	SSS5      /S.:X  d   eg )Nsk   a,b,c
1970,1970-01-01 00:00:00,1970-01-01 00:00:00.123
1989,1989-07-14 01:00:00,1989-07-14 01:00:00.123456
r   r   sr   ns  i  rN   r      r   ix @ rF  )r   r9   r   r;   	timestampr  r   r  s       r   test_simple_timestamps'BaseCSVTableRead.test_simple_timestamps  s    I %S"((*- ",,s"34 ",,t"457 8 ||v%%% 4A&q"a(@A4Aq!Q74B1a8:%
 
 	
 
rL   c                    Sn[        5       nU R                  XS9n[        R                  " S[        R                  " 5       4S[        R
                  " S5      4/5      nUR                  U:X  d   eUR                  5       SS/[        SS	S	5      [        SS	S
5      /S.:X  d   eS/Ul        U R                  XS9n[        R                  " S[        R
                  " S5      4S[        R                  " 5       4/5      nUR                  U:X  d   eUR                  5       [        SS	S	5      [        SS	S
5      /SS/S.:X  d   eS[        /Ul        U R                  XS9n[        R                  " S[        R
                  " S5      4S[        R
                  " S5      4/5      nUR                  U:X  d   eUR                  5       [        SS	S	5      [        SS	S
5      /[        SS	S	5      [        SS	S
5      /S.:X  d   eg )Ns6   a,b
1970/01/01,1980-01-01 00
1970/01/02,1980-01-02 00
r  r   r   r  z
1970/01/01z
1970/01/02i  rN   r   r   z%Y/%m/%dr  z1980-01-01 00z1980-01-02 00)
r
   r   r9   r   r   r  r  r   r   r   )ro   rn   rY   r   r   s        r   test_timestamp_parsers'BaseCSVTableRead.test_timestamp_parsers  s   K;S"))+. ",,s"346 7||v%%% -4A&q!(<=%
 
 	
 

 #-;S",,s"34 "))+.0 1||v%%% 4A&q!(<=!?3%
 
 	
 

 #-g!6;S",,s"34 ",,s"346 7||v%%% 4A&q!(<=4A&q!(<=%
 
 	
 
rL   c                    SnU R                  U5      n[        R                  " S[        R                  " 5       4S[        R                  " 5       4/5      nUR                  U:X  d   eUR	                  5       [        SSS5      [        SSS5      /[        SSS5      [        SSS5      /S.:X  d   e[        5       n[        R                  " 5       [        R                  " 5       S.Ul        U R                  XS	9n[        R                  " S[        R                  " 5       4S[        R                  " 5       4/5      nUR                  U:X  d   eUR	                  5       [        SSS5      [        SSS5      /[        SSS5      [        SSS5      /S.:X  d   e[        5       n[        R                  " S
5      [        R                  " S5      S.Ul        U R                  XS	9n[        R                  " S[        R                  " S
5      4S[        R                  " S5      4/5      nUR                  U:X  d   eUR	                  5       [        SSS5      [        SSS5      /[        SSS5      [        SSS5      /S.:X  d   eg )Ns0   a,b
1970-01-01,1970-01-02
1971-01-01,1971-01-02
r   r   r  rN   i  r   r   r  r  ms)r   r9   r   date32r  r   r
   date64r   r  r   )ro   rn   r   r   rY   s        r   
test_datesBaseCSVTableRead.test_dates*  s=   E%S"))+. "))+.0 1||v%%% tQ"Dq!$45tQ"Dq!$45%
 
 	
 
 "$))+BIIK@;S"))+. "))+.0 1||v%%% tQ"Dq!$45tQ"Dq!$45%
 
 	
 
 "$,,s"3",,t:LM;S",,s"34 ",,t"457 8||v%%% 4A&q!(<=4A&q!(<=%
 
 	
 
rL   c           	         SSK Jn  SnU R                  U5      n[        R                  " S[        R
                  " S5      4S[        R                  " 5       4/5      nUR                  U:X  d   eUR                  5       U" SSS	5      U" S
SS5      /SS/S.:X  d   e[        5       n[        R                  " S5      [        R
                  " S5      S.Ul
        U R                  X%S9n[        R                  " S[        R                  " S5      4S[        R
                  " S5      4/5      nUR                  U:X  d   eUR                  5       U" SSS	5      U" S
SS5      /U" SSS	S5      U" S
SSS5      /S.:X  d   eg )Nr   )times0   a,b
12:34:56,12:34:56.789
23:59:59,23:59:59.999
r   r  r      "   rs  r-  ;   z12:34:56.789z23:59:59.999r   usr  r  i
 iX> )r   r  r   r9   r   time32r   r  r
   time64r   )ro   r  rn   r   r   rY   s         r   
test_timesBaseCSVTableRead.test_timesN  sk   !E%S"))C.1 "))+.0 1||v%%% r2r"DR$45 .1%
 
 	
 
 "$))D/		$H;S"))D/2 "))D/24 5||v%%% r2r"DR$45r2r6*DRV,DE%
 
 	
 
rL   c                    [        SS9nSR                  5       nU R                  X!S9n[        R                  " S[        R
                  " [        R                  " 5       [        R                  " 5       5      4S[        R                  " 5       4/5      n/ SQ/ SQS	.nUR                  U:X  d   eUR                  5       U:X  d   eS
Ul
        U R                  X!S9nUR                  U:X  d   eUR                  5       U:X  d   eSUl
        U R                  X!S9nUR                  [        R                  " S[        R                  " 5       4S[        R                  " 5       4/5      :X  d   eUR                  5       U:X  d   eSUl
        SUl        SnU R                  X!SS9nUR                  U:X  d   eUS   R                  S5      R
                  n[        U5      S
:X  d   eUS   R                  5       S:X  d   eUS   R                  5       S:X  d   eSUl        U R                  X!S9n[        R                  " S[        R
                  " [        R                  " 5       [        R                   " 5       5      4S[        R                  " 5       4/5      n/ SQ/ SQS	.nUR                  U:X  d   eUR                  5       U:X  d   eg )NT)r   u   a,b
ab,1
cdé,2
cdé,3
ab,4r  r   r   )r      cdér  r   )rN   r   r   r   r   r   rN   2   Fs   a,b
ab,1
cd,2
ab,3)r'  rS  r   r      cd)   abr  r  rN   r   r   )r
   r8   r   r9   r   
dictionaryr   r   r;   r  r   r   chunkr   as_py	as_bufferr   )ro   rY   rn   r   r   rF   dict_valuess          r   test_auto_dict_encode&BaseCSVTableRead.test_auto_dict_encodei  s   t40779;S"--
BIIK"HI "((*-/ 0 .
 ||v%%% H,,,)*&;||v%%% H,,, *+&;||ryy3		*<+.
*;*=  > > 	> > H,,, *,&+.3   5||v%%%Cj&&q)44;1$$$1~##%---1~'')Y666 ;S"--
BIIK"HI "((*-/ 0 +
 ||v%%% H,,,rL   c                    [        SS/S9nSnU R                  X!S9n[        R                  " S[        R                  " 5       4S[        R
                  " 5       4S[        R
                  " 5       4S	[        R                  " 5       4/5      nUR                  U:X  d   eUR                  5       S S /SS
/SS/SS /S.:X  d   e[        SS/SS9nU R                  X!S9nUR                  5       S S /S S
/SS/SS /S.:X  d   eSUl        U R                  X!S9nUR                  5       S S /SS
/SS/SS /S.:X  d   e[        / S9nSnU R                  X!S9n[        R                  " S[        R
                  " 5       4S[        R
                  " 5       4/5      nUR                  U:X  d   eUR                  5       S
/S/S.:X  d   eg )NXxxZzz)r   s$   a,b,c,d
Zzz,"Xxx",1,2
Xxx,#N/A,,Zzz
r  r   r   r   r+  z#N/A1r   r   r*  T)r   r   Fs
   a,b
#N/A,
r   )	r
   r   r9   r   r   r   r;   r  r   ro   rY   rn   r   r   s        r   test_custom_nulls"BaseCSVTableRead.test_custom_nulls  s   5%.9=;S"''), "))+. "))+. "((*-/ 0 ||v%%% rT	%
 
 	
 
 5%.268; rT	%
 
 	
 
 +0'; rT	%
 
 	
 
 "-;S"))+. "))+.0 1||v%%% %
 
 	
 
rL   c                 V   [        SS/SS/S9nSnU R                  X!S9n[        R                  " S[        R                  " 5       4S	[        R
                  " 5       4S
[        R                  " 5       4/5      nUR                  U:X  d   eUR                  5       / SQ/ SQ/ SQS.:X  d   eg )Nr   yesr   no)r   r   s>   a,b,c
True,T,t
False,F,f
True,yes,yes
False,no,no
N/A,N/A,N/A
r  r   r   r   )TrueFalser  r  r   )TFTFN)tr  r  r  r   rF  )r
   r   r9   r   r   r  r  r  s        r   test_custom_bools"BaseCSVTableRead.test_custom_bools  s    3,,/;8! ;S"))+. "((*- "))+.0 1 ||v%%% :1/%
 
 	
 
rL   c                    [        SSS[        R                  " SS5      SS.S9nS	nU R                  X!S
9n[        R                  " S[        R
                  " 5       4S[        R                  " 5       4S[        R                  " 5       4S[        R                  " 5       4S[        R                  " SS5      4/5      nSS/SS/SS/SS/[        S5      [        S5      /S.nUR                  U:X  d   eUR                  5       U:X  d   e[        [        R                  " S[        R                  " 5       4S[        R                  " 5       4S[        R                  " 5       4S[        R                  " SS5      4S[        R                  " 5       4/5      S9nU R                  X!S
9nUR                  U:X  d   eUR                  5       U:X  d   eSn[        R                  " [        R                  5       nU R                  X!S
9  S S S 5        [        WR                  5      nSU;   d   eSU;   d   eg ! , (       d  f       N4= f)Nr   r   booleanr   r   r   )r   r   r+  r:  zzr   s(   a,b,c,d,e
1,2,3,true,1.0
4,-5,6,false,0
r  r   r   r   r+  r:  rN   r          @g      r  6TF1.00z0.00)r   r   r   r+  r:  r  s(   a,b,c,d,e
1,XXX,3,true,5
4,-5,6,false,7
zIn CSV column #1: z2CSV conversion error to float: invalid value 'XXX')r
   r9   
decimal128r   r   r;   r   r   r  r   r  rS   rT   r   r6   r^   )ro   rY   rn   r   r   rF   excerrs           r   test_column_types"BaseCSVTableRead.test_column_types  s   191:13r11E28	,: ;
 >;S"((*- "**,/ "))+. "((*- "--A"67	9 : Qts&/76?3
 ||v%%% H,,,S"**,$7%("))+$6%("((*$5%("--A*>$?%)288:$6	$8 9: ;||v%%% H,,,=]]2??+sOODO7 ,#))n#s***CsJJJ	 ,+s   2H00
H>c           
      B   S[         R                  " [         R                  " 5       [         R                  " 5       5      4S[         R                  " [         R                  " 5       [         R                  " 5       5      4S[         R                  " [         R                  " 5       [         R
                  " SS5      5      4S[         R                  " [         R                  " 5       [         R                  " 5       5      4/n[        [        U5      S9nSnU R                  X2S	9n[         R                  " U5      n/ S
Q/ SQ[        S5      [        S5      [        S5      // SQS.nUR                  U:X  d   eUR                  5       U:X  d   eS[         R                  " [         R                  " 5       [         R                  " 5       5      4US'   [        [        U5      S9n[        R                  " [         5         U R                  X2S	9nS S S 5        g ! , (       d  f       g = f)Nr   r   r   r   r   r+  r  s<   a,b,c,d
abc,123456,1.0,zz
defg,123456,0.5,xx
abc,N/A,1.0,xx
r  )r   defgr   )r  r  Nr  z0.50)r  r  r  r*  r   )r9   r  r   r   r;   r  
large_utf8r
   dictr   r   r   r  int8rS   rT   r   )ro   r   rY   rn   r   r   rF   s          r   test_column_types_dict'BaseCSVTableRead.test_column_types_dict  sr    "--
BGGI67"--
BHHJ78"--
BMM"a,@AB"--
BMMO<=	? 4+=>$ ;<(''&/76?GFOD#	
 ||v%%% H,,, bggi CDQ4+=>]]./OODO?E 0//s   7H
Hc                 \   Sn[        SS/S9n[        S[        R                  " 5       0S9nU R	                  XUS9n[        R
                  " S[        R                  " 5       4S[        R                  " 5       4/5      nUR
                  U:X  d   eUR                  5       / SQ/ SQS	.:X  d   eg )
Ns   a,b
c,d
e,f
r   r   rJ   r  r&  )   a   c   e)r   r+  r  r}  )r   r
   r9   r   r   r   r   r  r  s         r   #test_column_types_with_column_names4BaseCSVTableRead.test_column_types_with_column_names&  s     ""c
;(sBIIK6HI0?   AS"))+. "))+.0 1||v%%% # %
 
 	
 
rL   c                 j    SnU R                  U5      nUR                  5       SS/SS/SS/S.:X  d   eg )	N   a,b,c
1,2,3
4,5,6rN   r   r   r   r   r   rF  rl  rh  s      r   test_no_ending_newline'BaseCSVTableRead.test_no_ending_newline6  sH    %% QQQ%
 
 	
 
rL   c                 Z    SnU R                  U5      nUR                  5       S/ 0:X  d   eg )Ns   ,

r   rl  rh  s      r   test_trivialBaseCSVTableRead.test_trivial@  s.    % RH,,,rL   c                 :   SnU R                  U5      nUR                  5       SS/SS/S.:X  d   e[        SS9nU R                  XS	9nUR                  5       / S
Q/ SQS.:X  d   e[        SS9nU R                  XUS9nUR                  5       S S/S S/S.:X  d   eg )Ns   a,b
1,2

3,4
rN   r   r   r   r   F)r   r9  )NrN   Nr   )Nr   Nr   )r   rH  )r  2)r   r  r	   r   )ro   rn   r   r  r   s        r   test_empty_lines!BaseCSVTableRead.test_empty_linesF  s    +% QQ%
 
 	
 
 %>B ##%
 
 	
 
 #Q/-9   ; %
 
 	
 
rL   c                    Sn[         R                  " [        R                  SS9   U R	                  U5        S S S 5        Sn[         R                  " [        R                  SS9   U R	                  U5        S S S 5        S H@  n[         R                  " [        R                  SS9   U R	                  U5        S S S 5        MB     g ! , (       d  f       N= f! , (       d  f       Nf= f! , (       d  f       Mv  = f)Ns   a,b,c
1,2
4,5,6
r~  r   s   a,b,c
1,2,3
4zExpected 3 columns, got 1)rL   rp  rr  rq  s   

Empty CSV file)rS   rT   r9   r   r   )ro   rn   s     r   test_invalid_csv!BaseCSVTableRead.test_invalid_csv[  s    %]]2??2MNOOD! O!]]2??2MNOOD! O9Dr6FG% HG : ON ON HGs#   C%C'C(
C
C%(
C7	c                     SnU R                  U5      nUR                  5       S/S/S.:X  d   e[        SS9nU R                  XS9nUR                  5       S/S	/S
.:X  d   eg )N   a;b,c
de,fg;eh
defg;eha;br   r  r  r9  de,fgehr   b,c)r   r  r	   )ro   rn   r   rY   s       r   test_options_delimiter'BaseCSVTableRead.test_options_delimiterg  s    #% 6%
 
 	
 
 c*9 6%
 
 	
 
rL   c                     [        SSS9u  pU R                  U5      nUR                  UR                  :X  d   eUR                  U5      (       d   eUR	                  5       UR	                  5       :X  d   eg Nr   r   r  )rG   r   r   equalsr  )ro   rB   rF   r   s       r   test_small_random_csv&BaseCSVTableRead.test_small_random_csvu  sg    'R@$||x...||H%%%% H$6$6$8888rL   c                 L   [        SSS9u  p/ SQnXR                  S5      /nU H|  nU Hs  n[        US9nU R                  XWS9nUR                  UR                  :X  d   eUR                  U5      (       a  MO  UR                  5       UR                  5       :X  a  Ms   e   M~     g )Nr     r  )r   r        r  o   rr  r   r   )rG   rstripr   r   r   r  r  )	ro   csv_baserF   block_sizescsvsrB   r   r   r   s	            r   test_stress_block_sizes(BaseCSVTableRead.test_stress_block_sizes|  s    ,a#F///'23C)
*jAG||x666||H-- ??,0B0B0DDDD * rL   c                     [         R                  nSn[        U5       Vs/ s H  nSU 3PM
     nn[	        U5      nU" 5       n[        US S S2    Vs0 s H  ow[        R                  " 5       _M     snS9nU R                  XXS9n	U" 5       U-
  n
U
S::  d   eU	R                  U:X  d   eU	R                  S:X  d   eU	R                  U:X  d   eg ! [         a    [         R                   n Nf = fs  snf s  snf )Ni'  Kr   r  r        $@r   )r  thread_timerU   r   rK   r
   r9   r   r   r   r!   rJ   )ro   clockr   r"   rC   rB   t1kr'  r   dts              r   "test_stress_convert_options_blowup3BaseCSVTableRead.test_stress_convert_options_blowup  s    	$$E &+K&89&8qW&8	9Y'W(2;CaC.A.QRYY[..ACEWr\ Tzz  K///~~"""!!Y...#  	IIE	 : Bs   C C/C4C,+C,c           	         [         R                  " 5       R                  [         R                  " 5       R                  :w  a  [        R
                  " S5        S nSnSnSnUS:  a  US:  a  [        SU5        S	S
U-  -   nS n [        R                  " 5        n [         R                  " US9nUR                  5         [        R                  " 5       n	 U R                  U5        [        R                  " U// U/S5         S S S 5        Ub  UR                  b  OUS-  nUS:  a  US:  a  M  Wc  [        R                   " S5        US::  d   eUR                  n
[#        U
[$        R&                  5      (       d   eU
R(                  [*        R,                  :X  d   eg ! [         a$  n
U
n[        R                  " 5       U	-
  n S n
A
NS n
A
ff = f! [        R                  " U// U/S5        f = f! , (       d  f       N= f! [         a     GNf = f)Nz'test only works from main Python threadc                  x    [         R                  " S5        [        R                  " [        R                  5        g )Ng?)r  sleepsignalraise_signalSIGINTrO   rL   r   signal_from_thread>BaseCSVTableRead.test_cancellation.<locals>.signal_from_thread  s    JJsO.rL   g        i r   g      @r   zworkload size:s   a,b,c
s   1,2,3
targetr,  r   z0Failed to get an interruption during CSV readingr  )	threadingcurrent_threadidentmain_threadrS   r   printr   signal_wakeup_fdThreadstartr  r   KeyboardInterruptselect__context__failrx   r9   ArrowCancelledsignumr7  r9  )ro   r:  last_durationworkload_sizeattempts	large_csvexc_infosigfdr  r/  r:  s              r   test_cancellation"BaseCSVTableRead.test_cancellation  s   $$&,,%%'--.KKAB	/ c!hm"M2"Z-%??IH **,B%,,4FG	!YY[= OOI6 ugrE7DA -( # ''3 *A-MM c!hmP KKJK ###  !R..////xx6==(((?  1 ='(H,0IIK",<M= ugrE7DA -,  % sl   G> G-9GF%G-G> 
G
&G GG

GG**G--
G;7G> ;G> >
HHc                 v   ^  [         R                  " U 4S jS9nUR                  5         UR                  5         g )Nc                  &   > T R                  S5      $ )Ns   f64
0.1)r   r   s   r   <lambda>=BaseCSVTableRead.test_cancellation_disabled.<locals>.<lambda>  s    4??;7rL   r<  )r>  rD  rE  r4   )ro   r  s   ` r   test_cancellation_disabled+BaseCSVTableRead.test_cancellation_disabled  s*    
 79			rL   rO   N)*r   r   r   r   r   r   rd  ri  rm  rx  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r(  r2  rS   rO  r>  rR  rX  r   rO   rL   r   rR  rR    s    15 8	#21
f=2C:+
Z

 
.
2
" 
D"
H
6/-b,
\
*'KR@@
 
-
*
&
9E/0 [[@) @)D [[ rL   rR  c                   $    \ rS rSr\S 5       rSrg)TestSerialCSVTableReadi  c                     gNFrO   r   s    r   r   "TestSerialCSVTableRead.use_threads      rL   rO   Nr   r   r   r   rM  r   r   rO   rL   r   r[  r[         rL   r[  c                   $    \ rS rSr\S 5       rSrg)TestThreadedCSVTableReadi  c                     gNTrO   r   s    r   r   $TestThreadedCSVTableRead.use_threads      rL   rO   Nr`  rO   rL   r   rc  rc    s     rL   rc  c                      ^  \ rS rSrS rS rS rS rS rS r	S r
S	 rS
 rS rS rS rS rS rS rS rS rU 4S jr\R.                  R0                  U 4S j5       rSrU =r$ )BaseStreamingCSVReadi  c                 v    UR                  S[        5       5      nU R                  Ul        [        U/UQ70 UD6$ )z
Reads the CSV file into memory using pyarrow's open_csv
csv The CSV bytes
args Positional arguments to be forwarded to pyarrow's open_csv
kwargs Keyword arguments to be forwarded to pyarrow's open_csv
r   )rX  r   r   r   )ro   rB   r!  r   r   s        r   r   BaseStreamingCSVRead.open_csv  s;     ((G#'#3#3 -d-f--rL   c                 P    U R                   " [        R                  " U5      40 UD6$ r   )r   r9   r[  r   s      r   
open_bytesBaseStreamingCSVRead.open_bytes	  r]  rL   c                    UR                   U:X  d   e[        U5      n[        U5      [        U5      :X  d   e[        XC5       H<  u  pVUR	                  SS9  UR                   U:X  d   eUR                  5       U:X  a  M<   e   g )NTrU  )r   r0   r   zipr   r  )ro   readerexpected_schemara  batchesbatchexpected_batchs          r   check_reader!BaseStreamingCSVRead.check_reader  s|    }}///v,7|s=1111%(%@!ENNN%<<?222??$666 &ArL   c                 D    U R                   " U40 UD6R                  5       $ r   )rm  read_allr   s      r   r   BaseStreamingCSVRead.read_bytes  s    q+F+4466rL   c                    SnSS/SS/S.n[         R                  " U5      nU R                  U5      n[        R                  " S[        R
                  " 5       4S[        R
                  " 5       4/5      nU R                  XEU/5        g )	Ns   a,b
1,2
3,4
rN   r   r   r   r   r   r   )r.   r_  r   r9   r   r;   rv  )ro   r`  ra  rb  rq  rr  s         r   rd  %BaseStreamingCSVRead.test_file_object  su    ! !fAq62jjs#))c288:%6&)288:%6%8 9&M?CrL   c                     SnU R                  U5      n[        R                  " S[        R                  " 5       4S[        R                  " 5       4S[        R                  " 5       4/5      nU R	                  X#/ 5        g )Nrg  r   r   r  )rm  r9   r   r   rv  ro   rn   rq  rr  s       r   ri   BaseStreamingCSVRead.test_header!  sc    &))eRWWY%7&+RWWY%7&*BGGI%6%8 9 	&26rL   c                    Sn[         R                  " S[         R                  " 5       4S[         R                  " 5       4/5      n[	        5       n[        U5      Ul        U R                  XS9nU R                  XB/ SQ/ SQS./5        [        U5      S-
  Ul        U R                  XS9nU R                  XBS	S
/SS/S.S/S/S./5        g )N   a,b
123,456
abc,de
gh,ij
r   r   r   )123r   r  )   456   de   ijr   rN   r  r   r  r  r  r  )	r9   r   r   r   r   r   r   rm  rv  )ro   rn   rr  r   rq  s        r   test_inference#BaseStreamingCSVRead.test_inference)  s    3))c299;%7&)299;%7%9 : #}"%d)A&!5!;= >	? #&d)a-A&"'"()!46"&"'*+	,rL   c                 h   Sn[        5       n[        U5      S-
  Ul        U R                  XS9n[        R
                  " S[        R                  " 5       4S[        R                  " 5       4/5      nUR
                  U:X  d   eUR                  5       R                  5       S/S/S.:X  d   e[        R                  " [        S	S
9   UR                  5         S S S 5        [        R                  " [        5         UR                  5         S S S 5        g ! , (       d  f       NB= f! , (       d  f       g = f)Nr  r   r   r   r   {   i  r   zCSV conversion error to int64r   )r   r   r   rm  r9   r   r;   read_next_batchr  rS   rT   r   StopIterationro   rn   r   rq  rr  s        r   test_inference_failure+BaseStreamingCSVRead.test_inference_failure>  s    3"}"%d)a-A))c288:%6&)288:%6%8 9}}///%%'113cU8
 
 	
 
 ]]:!@B""$B ]]=)""$ *)	B B *)s   D8D#
D #
D1c                 f   Sn[        5       nSUl        [        R                  " [        R
                  SS9   U R                  XS9nS S S 5        SnSUl        U R                  XS9nUR                  5       R                  5       S/S	/S
.:X  d   e[        R                  " [        R
                  SS9   UR                  5         S S S 5        [        R                  " [        5         UR                  5         S S S 5        g ! , (       d  f       N= f! , (       d  f       NS= f! , (       d  f       g = f)Ns   a,b
1,2,3
4,5
6,7
r   zExpected 2 columns, got 3r   r   s   a,b
1,2
3,4,5
6,7
r   rN   r   r   )
r   r   rS   rT   r9   r   rm  r  r  r  )ro   rn   r   rq  s       r   r  %BaseStreamingCSVRead.test_invalid_csvR  s    ("}"$]]2??!<>__ % 1F> )"#A%%'113aSs7KKKK]]2??!<>""$> ]]=)""$ *)> >> > *)s#   D 3D&D" 
D
D"
D0c                    SnU R                  U5      n[        R                  " S[        R                  " 5       4S[        R                  " 5       4/5      nU R	                  X#S/S/S./5        [        SS9nU R                  XS	9n[        R                  " S
[        R                  " 5       4S[        R                  " 5       4/5      nU R	                  X#S/S/S./5        g )Nr  r  r   r  r  r  r  r  r9  r   r  r  r  r  )rm  r9   r   r   rv  r	   )ro   rn   rq  rr  rY   s        r   r  +BaseStreamingCSVRead.test_options_delimiterh  s    #&))eRYY[%9&)299;%7%9 :&$(6"), -	. c*:))c299;%7&+RYY[%9%; <&")$(6+ ,	-rL   c                    SnU R                  U5      n[        R                  " S[        R                  " 5       4S[        R                  " 5       4S[        R                  " 5       4/5      nU R	                  X#SS/SS/S	S
/S./5        g )Nr   r   r   r   rN   r   r   r   r   r   rF  )rm  r9   r   r;   rv  r~  s       r   r  +BaseStreamingCSVRead.test_no_ending_newliney  s    %&))c288:%6&)288:%6&)288:%6%8 9 	&"#Q"#Q"#Q) *	+rL   c                     [         R                  " [        SS9   U R                  S5        S S S 5        g ! , (       d  f       g = f)Nr  r   rL   )rS   rT   r   rm  r   s    r   test_empty_file$BaseStreamingCSVRead.test_empty_file  s)    ]]:-=>OOC  ?>>s	   5
Ac                 `   Sn[        5       n/ SQUl        U R                  XS9n[        R                  " S[        R
                  " 5       4S[        R
                  " 5       4S[        R
                  " 5       4/5      nU R                  X4SS/S	S
/SS/S./5        [        5       nSS/Ul        U R                  XUS9n[        R                  " S[        R
                  " 5       4S[        R
                  " 5       4/5      nU R                  X4S	S
/SS/S./5        S[        R                  " 5       0Ul
        U R                  XUS9n[        R                  " S[        R
                  " 5       4S[        R                  " 5       4/5      nU R                  X4SS/SS/S./5        / SQUl        [        R                  " [        SS9   U R                  XUS9nS S S 5        SUl        U R                  XUS9n[        R                  " S[        R                  " 5       4S[        R
                  " 5       4S[        R                  " 5       4/5      nU R                  X4S S /SS/SS/S./5        [        R                  " 5       [        R                   " 5       S.Ul
        U R                  XUS9n[        R                  " S[        R                   " 5       4S[        R
                  " 5       4S[        R                  " 5       4/5      nU R                  X4S S /SS/SS/S./5        g ! , (       d  f       GNQ= f)Ns   1,2,3
4,5,6)r+  r:  r  r   r+  r:  r  rN   r   r   r   r   r   r&  )r:  r  r  5)rD  r  r:  z,Column 'g' in include_columns does not existr   TrD  )rD  r:  r  )r:  rD  )r   rJ   rm  r9   r   r;   rv  r
   r   r   r   rS   rT   r  r   r   r  )ro   rn   r   rq  rr  r'  s         r   test_column_options(BaseStreamingCSVRead.test_column_options  s   "}$3!A))c288:%6&)288:%6&)288:%6%8 9 	&"#Q"#Q"#Q) *	+ )*+.*'1@ ! B))c288:%6&)288:%6%8 9&"#Q"#Q) *	+
 ),RYY['9$1@ ! B))c288:%6&)299;%7%9 :&"%s"#Q) *	+
 +:']]DF __T5D % FFF 37/1@ ! B))c2779%5&)288:%6&)299;%7%9 : 	&"&"%s"#Q) *	+
 .0YY[rzz|'L$1@ ! B))c2::<%8&)288:%6&)299;%7%9 : 	&"&"%s"#Q) *	+/F Fs   L
L-c                    Sn[        5       nU R                  XS9n[        R                  " S[        R                  " 5       4S[        R
                  " 5       4/5      nU R                  X4S/S/S./5        SUl        U R                  XS9n[        R                  " S[        R                  " 5       4S[        R                  " 5       4/5      nU R                  X4S/S	/S./5        S
nSUl        U R                  XS9n[        R                  " S[        R                  " 5       4S[        R                  " 5       4/5      nU R                  X4S/S	/S./5        g )Ns   a,b
un,lphantr   r   r   uns   lphantr   latin1u
   éléphants    a , b 
 u n ,  l  p h a n t r   )r   rm  r9   r   r   r   rv  r   r  s        r   test_encoding"BaseStreamingCSVRead.test_encoding  sO   ("}A))c299;%7&)299;%7%9 :&"&"3!46 7	8 !)A))c299;%7&)299;%7%9 :&"&".1 2	3
F 'A))c299;%7&)299;%7%9 :&"&".1 2	3rL   c                    [        SSS9u  pU R                  U5      nUR                  5       nUR                  UR                  :X  d   eUR	                  U5      (       d   eUR                  5       UR                  5       :X  d   eg r  )rG   rm  ry  r   r  r  )ro   rB   rF   rq  r   s        r   r  *BaseStreamingCSVRead.test_small_random_csv  ss    'R@%!||x...||H%%%% H$6$6$8888rL   c                    [        SSS9u  p/ SQnXR                  S5      /nU H  nU H  nUS U R                  S5      S:  d   e[        US9nU R	                  XWS9nUR                  5       n	U	R                  UR                  :X  d   eU	R                  U5      (       a  My  U	R                  5       UR                  5       :X  a  M   e   M     g )	Nr   r   r  )      r-  r  r  r#  rr  rp  r   r   )	rG   r$  countr   rm  ry  r   r  r  )
ro   r%  rF   r&  r'  rB   r   r   rq  r   s
             r   r(  ,BaseStreamingCSVRead.test_stress_block_sizes  s    ,a#F///'23C)
;J'--e4999*jA ) 4)||x666||H-- ??,0B0B0DDDD * rL   c                    [         R                  " 5         [        R                  " 5       nS nSn[	        5       nSS/Ul        SUl        U R                  X4S9nU" US/S/S	.5        [        R                  " 5       nU" US
/S/S	.5        [        R                  " 5       U::  d   eU" US/S/S	.5        [        R                  " 5       U::  d   eU" US/S/S	.5        [        R                  " 5       U::  d   e[        R                  " [        5         UR                  5         S S S 5        [        R                  " 5       U:X  d   eS n[        R                  " 5       U:X  d   eg ! , (       d  f       NG= f)Nc                 P    U R                  5       nUR                  5       U:X  d   eg r   )r  r  )rq  rF   rt  s      r   check_one_batchABaseStreamingCSVRead.test_batch_lifetime.<locals>.check_one_batch  s%    **,E??$000rL   s   10,11
12,13
14,15
16,17
r   r   r   r   r   r   r   r  r!  r  rG     r"  )gccollectr9   total_allocated_bytesr   rJ   r   rm  rS   rT   r  r  )ro   old_allocatedr  rn   r   rq  allocated_after_first_batchs          r   test_batch_lifetime(BaseStreamingCSVRead.test_batch_lifetime  sL   


002	1 /"}%(#J!"#Ard" 67&(&>&>&@#rd" 67'')-HHHHrd" 67'')-HHHHrd" 67'')-HHHH]]=)""$ *'')]:::'')]:::	 *)s   E##
E1c                   > [         TU ]  5         Sn[        5       nSUl        SS/Ul        U R                  XS9n[        R                  " [        5         UR                  5       (       d   e S S S 5        g ! , (       d  f       g = f)Nr   r   r   r   r   )
superr
  r   r   rJ   rm  rS   rT   r  r  ro   rn   rY   rq  	__class__s       r   r
  *BaseStreamingCSVRead.test_header_skip_rows  sl    %'. }!4L9]]=)))++++ *))s   A77
Bc                   > [         TU ]  5         Sn[        5       nSUl        U R	                  XS9n[
        R                  " [        5         UR                  5       (       d   e S S S 5        SUl        U R	                  XS9n[
        R                  " [        5         UR                  5       (       d   e S S S 5        g ! , (       d  f       N`= f! , (       d  f       g = f)Nr   r   r   r   )	r  r  r   r   rm  rS   rT   r  r  r  s       r   r  /BaseStreamingCSVRead.test_skip_rows_after_names+  s    *,. }%&"9]]=)))++++ * &+"9]]=)))++++ *) *) *)s   B?C?
C
CrO   )r   r   r   r   r   rm  rv  r   rd  ri  r  r  r  r  r  r  r  r  r  r(  r  r
  rS   rO  rP  r  r   __classcell__r  s   @r   ri  ri    s~    	.877D7,*%(%,-"
+!?+B3<9E$;8, [[, ,rL   ri  c                   $    \ rS rSr\S 5       rSrg)TestSerialStreamingCSVReadi?  c                     gr]  rO   r   s    r   r   &TestSerialStreamingCSVRead.use_threads@  r_  rL   rO   Nr`  rO   rL   r   r  r  ?  ra  rL   r  c                   $    \ rS rSr\S 5       rSrg)TestThreadedStreamingCSVReadiE  c                     gre  rO   r   s    r   r   (TestThreadedStreamingCSVRead.use_threadsG  rg  rL   rO   Nr`  rO   rL   r   r  r  E  s     rL   r  c                   ,    \ rS rSrS rS rS rS rSrg)BaseTestCompressedCSVReadiL  c                 6    [         R                  " SS9U l        g )Nzarrow-csv-test-)prefix)tempfilemkdtemptmpdirr   s    r   setUpBaseTestCompressedCSVRead.setUpN  s    &&.?@rL   c                 D    [         R                  " U R                  5        g r   )shutilrmtreer  r   s    r   tearDown"BaseTestCompressedCSVRead.tearDownQ  s    dkk"rL   c                      [        U5      $ ! [        R                   a)  n[        R                  " [        U5      5         S nAg S nAff = fr   )r   r9   ArrowNotImplementedErrorrS   r   r6   )ro   csv_pathr:  s      r   r   "BaseTestCompressedCSVRead.read_csvT  s;    	 H%%** 	 KKA	 s   
 A
AA
c                    [        SSS9u  p[        R                  R                  U R                  U R
                  5      nU R                  X15        U R                  U5      nUR                  SS9  UR                  UR                  :X  d   eUR                  U5      (       d   eUR                  5       UR                  5       :X  d   eg )Nr   r  r  TrU  )rG   ospathr4   r  csv_filename
write_filer   r   r   r  r  )ro   rB   rF   r  r   s        r   test_random_csv)BaseTestCompressedCSVRead.test_random_csvZ  s    'SA77<<T->->?&h'D!||x...||H%%%% H$6$6$8888rL   )r  N)	r   r   r   r   r  r  r   r  r   rO   rL   r   r  r  L  s    A# 9rL   r  c                   $    \ rS rSrSrS rS rSrg)TestGZipCSVReadie  zcompressed.csv.gzc                     [         R                  " USS5       nUR                  U5        S S S 5        g ! , (       d  f       g = f)Nwbr   )gzipopenr3   ro   r  contentsr  s       r   r  TestGZipCSVRead.write_fileh  s,    YYtT1%GGH &%%s	   4
Ac                    [         R                  R                  U R                  U R                  5      n[
        R                  " USS5       nUR                  S5        S S S 5        [
        R                  " USS5       nUR                  S5        S S S 5        U R                  U5      nUR                  5       / SQ/ SQS.:X  d   eg ! , (       d  f       No= f! , (       d  f       NN= f)	Nr  r   s   ab,cd
ef,gh
r   s   ij,kl
mn,op
r  r  r  )
r  r  r4   r  r  r  r  r3   r   r  )ro   r  r  r   s       r   test_concatenated!TestGZipCSVRead.test_concatenatedl  s    77<<T->->?YYxq)QGG%& *YYxq)QGG%& *h' $$%
 
 	
 
 *)))s   C?C
C
C'rO   N)r   r   r   r   r  r  r  r   rO   rL   r   r  r  e  s    &L
rL   r  c                       \ rS rSrSrS rSrg)TestBZ2CSVReadiz  zcompressed.csv.bz2c                     [         R                  " US5       nUR                  U5        S S S 5        g ! , (       d  f       g = f)Nr   )bz2BZ2Filer3   r  s       r   r  TestBZ2CSVRead.write_file}  s*    [[s#qGGH $##s	   3
ArO   N)r   r   r   r   r  r  r   rO   rL   r   r  r  z  s    'LrL   r  c                  l    [         R                  " S5      n [        U 5        U R                  (       a   eg Nr   )r.   r_  r   closed)bufs    r   0test_read_csv_does_not_close_passed_file_handlesr    s&    
**+
,CSMzz>zrL   c            
         [         R                  R                  / SQ/ SQ/SS/5      n U R                  SS9S   nX4 H  n[        R
                  " 5       n[        X#[        SS	95        UR                  S5        U [        U5      :X  d   e[        R
                  " 5       n[        X#[        S
S	95        UR                  S5        [        U R                  S9nU [        X4S9:X  a  M   e   S S [        SS	94[        U R                  S9S [        S
S	94S [        SS9[        SSS94[        U R                  S9[        SS9[        S
SS944 H  u  pEn[        R
                  " 5       n[        X0R                  US9 nUR                  U 5        S S S 5        UR                  S5        U [        X4US9:X  d   e[        R
                  " 5       n[        X0R                  US9 nU R                  SS9 H  nUR!                  U5        M     S S S 5        UR                  S5        U [        X4US9:X  a  M   e   g ! , (       d  f       N= f! , (       d  f       NE= f)Nr  rF  c1c2r   )max_chunksizer   T)r   Fr  r   r   r  )r   r   r   write_optionsr  rN   )r9   r<   r=   ru  r.   r_  r   r   seekr   r   rJ   r	   r   r   write_tablewrite_batch)	r  record_batchr`  r  r   r  r  writerrt  s	            r   test_write_read_round_tripr     s   
i9D$<HA<<a<03L!jjl$\>?HSM!!! jjl$\?@"?HS<<<< "" 
t\67	!..	14	U	+	-	|c*	TS	9	;	!..	1		%	Ud	;	=	72] jjlsHHMBfq! CHS+8: : 	: :jjlsHHMBfA6""5) 7 C 	HS+8: : 	: :+	7 CB CBs   9H3(I3
I	
I	c                     [         R                  R                  / SQ/ SQ/SS/5      n [        R                  " 5       n[        SS9S4[        5       S4[        S	S9S
44 HZ  u  p#[        XR                  US9 nUR                  U 5        S S S 5        UR                  5       U:X  d   eUR                  S5        M\     [         R                  R                  SS//S/5      n [        R                  " 5       n[        SS9S4[        SS9[         R                  R                  44 H\  u  p#[        XR                  US9 n UR                  U 5         S S S 5        UR                  5       U:X  d   eUR                  S5        M^     g ! , (       d  f       GN= f! [         a%  n[        XS5      (       d   e S nAS S S 5          g S nAff = f! , (       d  f       N~= f)NrN   r   Nr   Nr   r  r  r   )r   s   "c1","c2"
1,a
2,
,c
   "c1","c2"
1,"a"
2,
,"c"
r   s   "c1","c2"
"1","a"
"2",
,"c"
r  r   r)   r   r   s   "c1"
","
""""
)r9   r<   r=   r.   r_  r   r   r   r  r7   r  libr   	Exceptionrx   )r  r  r  resr  r:  s         r   test_write_quoting_styler    s   
l,<=d|LA
**,C	F	+-HI	89	K	0	,	. sHHMBfq! C||~$$$ 	sDk]TF3A
**,C	H	-/CD	F	+RVV-@-@A sHHMBf""1% C ||~$$$ CB   "!)))) CB CBs<   8F=G	?F
F	
G!G3G	GG		
G	c                     [         R                  R                  / SQ/ SQ/SS/5      n [        R                  " 5       n[        SS9S4[        5       S4[        S	S9S44 HZ  u  p#[        XR                  US
9 nUR                  U 5        S S S 5        UR                  5       U:X  d   eUR                  S5        M\     g ! , (       d  f       N:= f)Nr  r  r  r  r   )quoting_headers   c1,c2
1,"a"
2,
,"c"
r  r   r  r   )r9   r<   r=   r.   r_  r   r   r   r  r7   r  )r  r  r  r  r  s        r   test_write_quoting_headerr    s    
l,<=d|LA
**,C	V	,.IJ	89	[	1	(	* sHHMBfq! C||~$$$ CBs   8B>>
C	c                      S n [         R                  " 5          U " 5       nU" 5       b   e S S S 5        g ! , (       d  f       g = f)Nc                  p    [         R                  " S5      n [        U 5      n[        R                  " U5      $ r  )r.   r_  r   weakrefref)r  r   s     r   inner,test_read_csv_reference_cycle.<locals>.inner  s)    jj/0{{5!!rL   )r   disabled_gc)r  wrs     r   test_read_csv_reference_cycler    s4    "
 
			Wt|| 
		s	   5
Atype_factoryc                  0    [         R                  " SS5      $ N   rN   r9   r  rO   rL   r   rV  rV        BMM"a rL   c                  0    [         R                  " SS5      $ )N&   rG  r  rO   rL   r   rV  rV        BMM"b!rL   c                  0    [         R                  " SS5      $ r  r9   
decimal256rO   rL   r   rV  rV    r  rL   c                  0    [         R                  " SS5      $ )NL   r   r  rO   rL   r   rV  rV    r  rL   c                 .   U" 5       n[         R                  " S[         R                  " SS/5      R                  U5      05      n[	        X0S-  5        [        U S-  5      nUR                  S5      R                  U5      UR                  S5      :X  d   eg )NcolrN   r   zout.csv)r9   r   r:   castr   r   column)r  r  r*   r   outs        r   test_write_csv_decimalr(    s}     >DHHeRXXq!f-22489:Eei'(
6I%
&C::e!!$'5<<+>>>>rL   c                      Sn  " S S[         R                  5      n[        S5       HA  n[        R                  " [
        R                  5         [        U" U 5      5        S S S 5        MC     g ! , (       d  f       MU  = f)Ns   a,b,cc                   4   ^  \ rS rSrU 4S jrU 4S jrSrU =r$ )-test_read_csv_gil_deadlock.<locals>.MyBytesIOi  c                 J   > [         R                  " S5        [        TU ]  " U6 $ NgMbP?)r  r6  r  readro   r!  r  s     r   r.  2test_read_csv_gil_deadlock.<locals>.MyBytesIO.read	  s    JJu7<&&rL   c                 J   > [         R                  " S5        [        TU ]  " U6 $ r-  )r  r6  r  readintor/  s     r   r2  6test_read_csv_gil_deadlock.<locals>.MyBytesIO.readinto  s    JJu7#T**rL   rO   )r   r   r   r   r.  r2  r   r  r  s   @r   	MyBytesIOr+    s    	'	+ 	+rL   r4  r  )r.   r_  r   rS   rT   r9   r   r   )r`  r4  r"   s      r   test_read_csv_gil_deadlockr5    sT    
 D+BJJ + 2Y]]2??+Yt_% ,+ ++s   A--
A<	)r   r   z
T)Gr   r  r   r   decimalr   r  r  r.   r1   r  r+   rG  r  r7  r   r  r>  r  unittestr  rS   pyarrowr9   pyarrow.csvr   r   r   r	   r
   r   r   r   r   r   pyarrow.testsr   r   r#   r%   rG   rK   r_   rg   ri   r   r   r   r   r   ABCr   rR  r[  rc  ri  r  rO  r  r  TestCaser  r  r  r   r  r  r  parametrizer(  r5  rO   rL   r   <module>r>     s  $  
 #  	  	  	            4 4 4 !"$#,00- -$&4n9xN;b0XK#'' XKv
T{ Tn- / @,; @,F
!5  #7  9 92
/1B1B 
*.0A0A ):XB	  ! !	* ??&rL   