U
    `»{j?   ã                   @   s   d Z ddlZddlmZ ddlZddlZddlmZmZ ddl	Z
ddlZddlmZ ddlmZ ddlmZmZmZmZ ddlm  mZ ddlZddlmZ ddlmZ dd	l m!Z!m"Z"m#Z#m$Z$m%Z% zddl&Z&d
Z'W n e(k
rî   dZ'Y nX z>eƒ �* edde)d� edde*d� ddl+Z+W 5 Q R X d
Z,W n e(k
�rH   dZ,Y nX ej- d¡Z.ej/ej0dej-j1e, �pxedƒdkdd�d�ej0dej-j1e' dd�d�gd�dd„ ƒZ2ej/dd„ ƒZ3ej/dd„ ƒZ4ej/d d!„ ƒZ5ej/d"d#„ ƒZ6ej/d$d%„ ƒZ7ej/ej 8ej9j:¡ej 8ej9j;¡ej 8ej9j<¡ej =d&d'¡ej =d(d'¡ej =d)d'¡ej =d*d'¡gd�d+d,„ ƒZ>dKd.d/„Z?d0d1„ Z@d2d3„ ZAd4d5„ ZBd6d7„ ZCd8d9„ ZDd:d;„ ZEd<d=„ ZFd>d?„ ZGd@dA„ ZHG dBdC„ dCƒZIG dDdE„ dEeIƒZJej- dF¡G dGdH„ dHeIƒƒZKG dIdJ„ dJeIƒZLdS )Lz test parquet compat é    N)ÚBytesIO)Úcatch_warningsÚfilterwarnings)Ú
get_option)Úis_platform_windows)Úpa_version_under2p0Úpa_version_under5p0Úpa_version_under6p0Úpa_version_under8p0)ÚVersion)ÚFastParquetImplÚPyArrowImplÚ
get_engineÚread_parquetÚ
to_parquetTFÚignorez	`np.bool`)Úcategoryz.*Int64Index.*z4ignore:RangeIndex.* is deprecated:DeprecationWarningÚfastparquetúmode.data_managerÚarrayz4fastparquet is not installed or ArrayManager is used©Úreason)ZmarksÚpyarrowúpyarrow is not installed)Úparamsc                 C   s   | j S ©N©Úparam©Úrequest© r    ú[/var/www/html/bwave/python/venv/lib/python3.8/site-packages/pandas/tests/io/test_parquet.pyÚengineB   s    r"   c                   C   s   t st d¡ dS )Nr   r   )Ú_HAVE_PYARROWÚpytestÚskipr    r    r    r!   ÚpaW   s    
r&   c                   C   s*   t st d¡ ntdƒdkr&t d¡ dS )Nzfastparquet is not installedr   r   z.ArrayManager is not supported with fastparquetr   )Ú_HAVE_FASTPARQUETr$   r%   r   r    r    r    r!   Úfp^   s
    
r(   c                   C   s   t  dddgddœ¡S )Né   é   é   Úfoo)ÚAÚB)ÚpdÚ	DataFramer    r    r    r!   Ú	df_compatg   s    r1   c               
   C   sD   t  tdƒttddƒƒtjdddd�dd	dgt jd
dd�dœ¡} | S )NÚabcr)   é   ç      @ç      @Úfloat64©ÚdtypeTFÚ20130101r+   ©Úperiods)ÚaÚbÚdÚeÚf)r/   r0   ÚlistÚrangeÚnpÚarangeÚ
date_range)Údfr    r    r!   Údf_cross_compatl   s    úÿrG   c                   C   s’   t  tdƒdtjdgdd dgdddgddd	gttd
dƒƒt dd¡ d¡tjdddd�dtjdgdddgt jddd�t  	d¡t j
t  	d¡gdœ¡S )Nr2   r<   Úcó   fooó   bars   bazr,   ÚbarÚbazr)   r3   r+   é   Úu1r4   r5   r6   r7   ç       @ç      @TFr9   r:   Z20130103)ÚstringZstring_with_nanZstring_with_noneÚbytesÚunicodeÚintZuintÚfloatZfloat_with_nanÚboolÚdatetimeZdatetime_with_nat)r/   r0   rA   rC   ÚnanrB   rD   ÚastyperE   Ú	TimestampZNaTr    r    r    r!   Údf_full~   s$    

ýôÿr[   z2019-01-04T16:41:24+0200z%Y-%m-%dT%H:%M:%S%zz2019-01-04T16:41:24+0215z2019-01-04T16:41:24-0200z2019-01-04T16:41:24-0215c                 C   s   | j S r   r   r   r    r    r!   Útimezone_aware_date_list–   s    r\   r*   c
              	      s€   ˆp
ddi‰ˆpi ‰ˆdkr ˆ‰|r4|ˆd< |ˆd< ‡ ‡‡‡‡‡‡‡fdd„}
ˆdkrtt  ¡ �‰|
|	ƒ W 5 Q R X n|
|	ƒ dS )a  Verify parquet serializer and deserializer produce the same results.

    Performs a pandas to disk and disk to pandas round trip,
    then compares the 2 resulting DataFrames to verify equality.

    Parameters
    ----------
    df: Dataframe
    engine: str, optional
        'pyarrow' or 'fastparquet'
    path: str, optional
    write_kwargs: dict of str:str, optional
    read_kwargs: dict of str:str, optional
    expected: DataFrame, optional
        Expected deserialization result, otherwise will be equal to `df`
    check_names: list of str, optional
        Closed set of column names to be compared
    check_like: bool, optional
        If True, ignore the order of index & columns.
    repeat: int, optional
        How many times to repeat the test
    ÚcompressionNr"   c              
      sV   t | ƒD ]H}ˆjˆfˆŽ tdd�� tˆfˆŽ}W 5 Q R X tjˆ|ˆˆˆ d� qd S )NT©Úrecord)Úcheck_namesÚ
check_likeÚcheck_dtype)rB   r   r   r   ÚtmÚassert_frame_equal)ÚrepeatÚ_Úactual©rb   ra   r`   rF   ÚexpectedÚpathÚread_kwargsÚwrite_kwargsr    r!   ÚcompareÑ   s    ûz!check_round_trip.<locals>.compare)rc   Úensure_clean)rF   r"   rj   rl   rk   ri   r`   ra   rb   re   rm   r    rh   r!   Úcheck_round_trip¥   s    "
ro   c                 C   s|   t rLddlm} |j| dd�}t|jjƒt|ƒks6t‚|jjt|ƒksxt‚n,ddl	m
} |j
| dd�}|jjj|ksxt‚dS )zÂCheck partitions of a parquet file are as expected.

    Parameters
    ----------
    path: str
        Path of the dataset.
    expected: iterable of str
        Expected partition names.
    r   NF)Zvalidate_schemaZhive)Úpartitioning)r   Úpyarrow.parquetÚparquetZParquetDatasetÚlenZ
partitionsZpartition_namesÚAssertionErrorÚsetZpyarrow.datasetÚdatasetrp   ÚschemaÚnames)rj   ri   Úpqrv   Zdsr    r    r!   Úcheck_partition_namesæ   s    
rz   c              	   C   s.   d}t jt|d�� t| ddƒ W 5 Q R X d S )Nz.engine must be one of 'pyarrow', 'fastparquet'©Úmatchr,   rK   )r$   ÚraisesÚ
ValueErrorro   )r1   Úmsgr    r    r!   Útest_invalid_engineý   s    r€   c              	   C   s$   t  dd¡� t| ƒ W 5 Q R X d S )Núio.parquet.enginer   ©r/   Úoption_contextro   )r1   r&   r    r    r!   Útest_options_py  s    r„   c              	   C   s$   t  dd¡� t| ƒ W 5 Q R X d S )Nr�   r   r‚   )r1   r(   r    r    r!   Útest_options_fp
  s    r…   c              	   C   s$   t  dd¡� t| ƒ W 5 Q R X d S )Nr�   Úautor‚   )r1   r(   r&   r    r    r!   Útest_options_auto  s    r‡   c              	   C   s  t tdƒtƒst‚t tdƒtƒs$t‚t dd¡�< t tdƒtƒsDt‚t tdƒtƒsVt‚t tdƒtƒsht‚W 5 Q R X t dd¡�< t tdƒtƒs’t‚t tdƒtƒs¤t‚t tdƒtƒs¶t‚W 5 Q R X t dd¡�> t tdƒtƒsàt‚t tdƒtƒsòt‚t tdƒtƒ�st‚W 5 Q R X d S )Nr   r   r�   r†   )Ú
isinstancer   r   rt   r   r/   rƒ   )r(   r&   r    r    r!   Útest_options_get_engine  s    r‰   c               	   C   s.  ddl m}  |  d¡}|  d¡}ts(dnttjƒt|ƒk }tsBdnttjƒt|ƒk }to\| }tof| }|�s*|�s*|r¨d|› d�}t	j
t|d�� td	ƒ W 5 Q R X n&d
}t	j
t|d�� td	ƒ W 5 Q R X |�rd|› d�}t	j
t|d�� td	ƒ W 5 Q R X n&d}t	j
t|d�� td	ƒ W 5 Q R X d S )Nr   )ÚVERSIONSr   r   FzPandas requires version .z. or newer of .pyarrow.r{   r†   z%Missing optional dependency .pyarrow.z. or newer of .fastparquet.z)Missing optional dependency .fastparquet.)Zpandas.compat._optionalrŠ   Úgetr#   r   r   Ú__version__r'   r   r$   r}   ÚImportErrorr   )rŠ   Z
pa_min_verZ
fp_min_verZhave_pa_bad_versionZhave_fp_bad_versionZhave_usable_paZhave_usable_fpr|   r    r    r!   Ú"test_get_engine_auto_error_message,  s8    

ÿýÿý

rŽ   c              	   C   sj   | }t  ¡ �T}|j||d d� t||d�}t  ||¡ t||ddgd�}t  ||ddg ¡ W 5 Q R X d S )N©r"   r]   ©r"   r<   r>   ©r"   Úcolumns)rc   rn   r   r   rd   ©rG   r&   r(   rF   rj   Úresultr    r    r!   Útest_cross_engine_pa_fpW  s    
r•   c              
   C   s€   | }t  ¡ �j}|j||d d� tdd��D t||d�}t  ||¡ t||ddgd�}t  ||ddg ¡ W 5 Q R X W 5 Q R X d S )Nr�   Tr^   r�   r<   r>   r‘   )rc   rn   r   r   r   rd   r“   r    r    r!   Útest_cross_engine_fp_pae  s    
r–   c                   @   s:   e Zd Zdd„ Zdd„ Zejjejddd�dd	„ ƒƒZ	d
S )ÚBasec              
   C   sB   t  ¡ �0}tj||d�� t|||d d� W 5 Q R X W 5 Q R X d S )Nr{   ©r]   )rc   rn   r$   r}   r   )ÚselfrF   r"   ÚexcÚerr_msgrj   r    r    r!   Úcheck_error_on_writet  s    
zBase.check_error_on_writec              
   C   s>   t  ¡ �,}t  |¡� t|||d d� W 5 Q R X W 5 Q R X d S )Nr˜   )rc   rn   Úexternal_error_raisedr   )r™   rF   r"   rš   rj   r    r    r!   Úcheck_external_error_on_writez  s    
z"Base.check_external_error_on_writeúdhttps://raw.githubusercontent.com/pandas-dev/pandas/main/pandas/tests/io/data/parquet/simple.parquetT)ÚurlZcheck_before_testc                 C   s.   |dkrt  |¡ d}t|ƒ}t ||¡ d S )Nr†   rŸ   )r$   Úimportorskipr   rc   rd   )r™   r1   r"   r    rF   r    r    r!   Útest_parquet_read_from_url€  s    	
ÿzBase.test_parquet_read_from_urlN)
Ú__name__Ú
__module__Ú__qualname__rœ   rž   r$   ÚmarkÚnetworkrc   r¢   r    r    r    r!   r—   s  s   ûr—   c                   @   sÂ   e Zd Zdd„ Zdd„ Zdd„ Zej ddd	d
dg¡dd„ ƒZ	dd„ Z
dd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zd d!„ Zd"d#„ Zej d$d%d&d'd(d)d*d+d,d-g	¡d.d/„ ƒZdS )0Ú	TestBasicc              	   C   sJ   t  dddg¡ddt  d¡t dddg¡fD ]}d}|  ||t|¡ q,d S )Nr)   r*   r+   r,   r9   z+to_parquet only supports IO with DataFrames)r/   ÚSeriesrZ   rC   r   rœ   r~   )r™   r"   Úobjr   r    r    r!   Ú
test_error”  s    ûzTestBasic.test_errorc                 C   s6   t  tdƒttddƒƒdœ¡}ddg|_t||ƒ d S )Nr2   r)   r3   ©rQ   rT   r,   rK   )r/   r0   rA   rB   r’   ro   )r™   r"   rF   r    r    r!   Útest_columns_dtypesŸ  s    
zTestBasic.test_columns_dtypesc                 C   s�   t  tdƒttddƒƒdœ¡}d}ddg|_|  ||t|¡ ddg|_|  ||t|¡ t d	dddd¡t d	dddd¡g|_|  ||t|¡ d S )
Nr2   r)   r3   r¬   ú%parquet must have string column namesr   rI   rJ   iÛ  )r/   r0   rA   rB   r’   rœ   r~   rW   )r™   r"   rF   r   r    r    r!   Útest_columns_dtypes_invalid¦  s    

þz%TestBasic.test_columns_dtypes_invalidr]   NÚgzipÚsnappyÚbrotlic                 C   sP   |dkrt  d¡ n|dkr&t  d¡ t ddddgi¡}t||d|id� d S )	Nr±   r²   r-   r)   r*   r+   r]   ©rl   )r$   r¡   r/   r0   ro   )r™   r"   r]   rF   r    r    r!   Útest_compression¹  s    
zTestBasic.test_compressionc                 C   sJ   t  tdƒttddƒƒdœ¡}t  dtdƒi¡}t|||ddgid� d S )Nr2   r)   r3   r¬   rQ   r’   ©ri   rk   )r/   r0   rA   rB   ro   )r™   r"   rF   ri   r    r    r!   Útest_read_columnsÅ  s       ÿzTestBasic.test_read_columnsc                 C   s¦   |dk}t  ddddgi¡}t||ƒ dddgt jddd�td	ƒdddgg}|D ]2}||_t|t jƒrt|j d ¡|_t|||d
� qPdddg|_d|j_	t||ƒ d S )Nr   r-   r)   r*   r+   r3   r9   r:   r2   )r`   r   r,   )
r/   r0   ro   rE   rA   Úindexrˆ   ZDatetimeIndexÚ
_with_freqÚname)r™   r"   r`   rF   Zindexesr·   r    r    r!   Útest_write_indexÎ  s     
üzTestBasic.test_write_indexc                 C   s>   |}t  ddddgi¡}t j dddg¡}||_t||ƒ d S )Nr-   r)   r*   r+   ©r<   r)   ©r<   r*   ©r=   r)   )r/   r0   Ú
MultiIndexÚfrom_tuplesr·   ro   )r™   r&   r"   rF   r·   r    r    r!   Útest_write_multiindexæ  s
    zTestBasic.test_write_multiindexc                 C   s¢   |}t jdddd�}t jtj dt|ƒ d¡tdƒd�}t jj	d	d
g|gddgd�}|j
d d�}||fD ]4}||_t||ƒ t||dddgi|ddg d� qhd S )Nz01-Jan-2018z01-Dec-2018ZMS)Úfreqr*   r+   ÚABC©r’   ÚLevel1ÚLevel2ÚlevelÚdate)rx   r’   r-   r.   ©rk   ri   )r/   rE   r0   rC   ÚrandomÚrandnrs   rA   r¾   Zfrom_productÚcopyr·   ro   )r™   r&   r"   ÚdatesrF   Zindex1Úindex2r·   r    r    r!   Útest_multiindex_with_columnsï  s"    $
 ÿ
  
 
ÿz&TestBasic.test_multiindex_with_columnsc              	   C   sê   t  dddgdddgdœ¡}d dd	œ}|jd
d�}t||||d� t jdddgdddgdœdddgd�}t||||d� ddddddddgddddddddgg}t jttdƒƒdd„ tdƒD ƒdœ|d�}|jd
d�}t||||d� d S )Nr)   r*   r+   ÚqÚrÚs)r<   r=   F)r]   r·   T)Údrop©rl   ri   ZzyxZwvuZtsr)r·   rK   rL   r,   ÚquxÚoneÚtwoé   c                 S   s   g | ]
}| ‘qS r    r    )Ú.0Úir    r    r!   Ú
<listcomp>  s     z7TestBasic.test_write_ignoring_index.<locals>.<listcomp>)rÕ   rÖ   )r/   r0   Zreset_indexro   rA   rB   )r™   r"   rF   rl   ri   Úarraysr    r    r!   Útest_write_ignoring_indexÿ  s$    
 ÿþ ÿz#TestBasic.test_write_ignoring_indexc                 C   sB   t j dddg¡}t jtj dd¡|d�}d}|  ||t|¡ d S )Nr»   r¼   r½   r3   r+   rÃ   úY\s*parquet must have string column names for all values in\s*each level of the MultiIndex)	r/   r¾   r¿   r0   rC   rÉ   rÊ   rœ   r~   )r™   r"   Z
mi_columnsrF   r   r    r    r!   Útest_write_column_multiindex  s
    ÿz&TestBasic.test_write_column_multiindexc              	   C   sh   |}ddddddddgddddddddgg}t jtj dd¡|d�}d	d
g|j_d}|  ||t|¡ d S )NrK   rL   r,   rÔ   r)   r*   r×   rÃ   rÄ   rÅ   rÝ   )	r/   r0   rC   rÉ   rÊ   r’   rx   rœ   r~   ©r™   r&   r"   rÛ   rF   r   r    r    r!   Ú&test_write_column_multiindex_nonstring)  s    þÿz0TestBasic.test_write_column_multiindex_nonstringc              	   C   s^   |}ddddddddgddddddddgg}t jtj dd¡|d�}d	d
g|j_t||ƒ d S )NrK   rL   r,   rÔ   rÕ   rÖ   r×   rÃ   Z	ColLevel1Z	ColLevel2)r/   r0   rC   rÉ   rÊ   r’   rx   ro   ©r™   r&   r"   rÛ   rF   r    r    r!   Ú#test_write_column_multiindex_string;  s    þz-TestBasic.test_write_column_multiindex_stringc                 C   s>   |}ddddg}t jtj dd¡|d�}d|j_t||ƒ d S )	NrK   rL   r,   rÔ   r×   r3   rÃ   Z	StringCol)r/   r0   rC   rÉ   rÊ   r’   r¹   ro   rá   r    r    r!   Útest_write_column_index_stringJ  s
    z(TestBasic.test_write_column_index_stringc                 C   sH   |}ddddg}t jtj dd¡|d�}d|j_d}|  ||t|¡ d S )	Nr)   r*   r+   r3   r×   rÃ   ZNonStringColr®   )	r/   r0   rC   rÉ   rÊ   r’   r¹   rœ   r~   rß   r    r    r!   Ú!test_write_column_index_nonstringV  s    z+TestBasic.test_write_column_index_nonstringc           
      C   sÖ  dd l m} |dkr.tjjdd�}|j |¡ t t 	dddd gd¡t 	dddd gd	¡t 	d
ddd g¡t 	dddd g¡t 	ddddgd¡t 	dddd gd¡t 	dddd gd¡dœ¡}t
 ¡ �,}| ||¡ t||d�}t||dd�}W 5 Q R X |d
 jt d¡k�st‚t tj	dddd gdd�tj	dddd gdd�tj	d
ddd gdd�tj	dddd gdd�tj	ddddgdd�tj	dddd gdd�tj	dddd gdd�dœ¡}	|dk�rÆ|jddd�}|	jddd�}	t
 ||	¡ d S ) Nr   r   z.Fastparquet nullable dtype support is disabledr   r)   r*   r+   Zint64Zuint8r<   r=   rH   TFr3   ç      ð?rO   rP   Zfloat32r6   )r<   r=   rH   r>   r?   r@   Úgr�   ©r"   Úuse_nullable_dtypesÚInt64r7   ÚUInt8rQ   ÚbooleanZFloat32ÚFloat64)Zaxis)rq   rr   r$   r¦   ÚxfailÚnodeÚ
add_markerr   Útabler   rc   rn   Zwrite_tabler   r8   rC   rt   r/   r0   rÒ   rd   )
r™   r"   r   ry   r¦   rð   rj   Zresult1Zresult2ri   r    r    r!   Útest_use_nullable_dtypesb  sF    ÿ÷ÿ
ùÿ
z"TestBasic.test_use_nullable_dtypesr8   ré   rê   rë   Úobjectzdatetime64[ns, UTC]rU   z	period[D]rì   rQ   c                 C   sT   t  dt jg |d�i¡}d }|dkr<t  dt jg dd�i¡}t||ddi|d� d S )NÚvaluer7   rU   rì   rè   TrÈ   )r/   r0   r   ro   )r™   r&   r8   rF   ri   r    r    r!   Útest_read_empty_array“  s$     ÿÿ ÿÿ   ÿzTestBasic.test_read_empty_array)r£   r¤   r¥   r«   r­   r¯   r$   r¦   Úparametrizer´   r¶   rº   rÀ   rÎ   rÜ   rÞ   rà   râ   rã   rä   rñ   rô   r    r    r    r!   r¨   “  s<   
		 
1÷þr¨   z8ignore:CategoricalBlock is deprecated:DeprecationWarningc                   @   s”  e Zd Zdd„ Zdd„ Zdd„ Zdd„ Zd	d
„ Zdd„ Zdd„ Z	e
jjeƒ dd�e
j deejg¡dd„ ƒƒZdd„ Ze
jjdd„ ƒZe
jjdd„ ƒZe
jje d¡e
j ddgg g¡dd„ ƒƒƒZe d¡d d!„ ƒZe d¡d"d#„ ƒZd$d%„ Zd&d'„ Ze
j deejg¡d(d)„ ƒZd*d+„ Zd,d-„ Z e d¡d.d/„ ƒZ!ejdd0d1�d2d3„ ƒZ"ejdd4d1�d5d6„ ƒZ#d7d8„ Z$d9d:„ Z%ejdd0d1�d;d<„ ƒZ&d=d>„ Z'd?S )@ÚTestParquetPyArrowc                 C   sB   |}t jdddd�}| d ¡}||d< dd dg|d< t||ƒ d S )Nr9   r+   úEurope/Brussels©r;   ÚtzÚdatetime_tzTZbool_with_none)r/   rE   r¸   ro   )r™   r&   r[   rF   Údtir    r    r!   Ú
test_basic·  s    
zTestParquetPyArrow.test_basicc                 C   s<   |}t jdddd�|d< t|||ddg dddgid	� d S )
Nr9   r+   r÷   rø   rú   rQ   rT   r’   rµ   )r/   rE   ro   )r™   r&   r[   rF   r    r    r!   Útest_basic_subset_columnsÃ  s    

üz,TestParquetPyArrow.test_basic_subset_columnsc                 C   s:   |j |d�}t|tƒst‚t|ƒ}t|ƒ}t ||¡ d S )Nr�   )r   rˆ   rR   rt   r   r   rc   rd   )r™   r&   r[   Z	buf_bytesZ
buf_streamÚresr    r    r!   Ú*test_to_bytes_without_path_or_buf_providedÑ  s
    z=TestParquetPyArrow.test_to_bytes_without_path_or_buf_providedc                 C   s8   t jt d¡ dd¡tdƒd� ¡ }|  ||td¡ d S )Né   r3   r+   ÚaaarÃ   zDuplicate column names found©	r/   r0   rC   rD   ZreshaperA   rË   rœ   r~   ©r™   r&   rF   r    r    r!   Útest_duplicate_columnsÜ  s    $z)TestParquetPyArrow.test_duplicate_columnsc                 C   s:   t  dt jddd�i¡}tr,|  ||t¡ n
t||ƒ d S )Nr<   ú1 dayr+   r:   )r/   r0   Útimedelta_ranger
   rž   ÚNotImplementedErrorro   r  r    r    r!   Útest_timedeltaá  s    z!TestParquetPyArrow.test_timedeltac                 C   s(   t  ddddgi¡}|  ||tj¡ d S )Nr<   r)   rO   )r/   r0   rž   r   ÚArrowExceptionr  r    r    r!   Útest_unsupportedè  s    z#TestParquetPyArrow.test_unsupportedc                 C   s6   t jddt jd�}tj|dgd�}|  ||tj¡ d S )Nr*   é
   r7   Úfp16©Údatar’   )rC   rD   Úfloat16r/   r0   rž   r   r	  )r™   r&   r  rF   r    r    r!   Útest_unsupported_float16ï  s    z+TestParquetPyArrow.test_unsupported_float16zqPyArrow does not cleanup of partial files dumps when unsupported dtypes are passed to_parquet function in windowsr   Ú	path_typec              
   C   sx   t jddt jd�}tj|dgd�}t ¡ �D}||ƒ}t tj	¡� |j
||d� W 5 Q R X tj |¡rjt‚W 5 Q R X d S )Nr*   r  r7   r  r  )rj   r"   )rC   rD   r  r/   r0   rc   rn   r�   r   r	  r   Úosrj   Úisfilert   )r™   r&   r  r  rF   Úpath_strrj   r    r    r!   Ú test_unsupported_float16_cleanupö  s    
z3TestParquetPyArrow.test_unsupported_float16_cleanupc                 C   sx   t  ¡ }t  tdƒ¡|d< t jddddd dgt  dddg¡d�|d< t jddddddgddd	gd
d�|d< t||ƒ d S )NZabcdefr<   rK   r,   rL   r7   r=   rH   r>   T)Ú
categoriesÚordered)r/   r0   ÚCategoricalrA   ZCategoricalDtypero   r  r    r    r!   Útest_categorical  s    þ
  ÿ
z#TestParquetPyArrow.test_categoricalc                 C   s4   t  d¡}|jf |Ž}d|i}t||d||d� d S )NÚs3fsÚ
filesystemzpandas-test/pyarrow.parquet©rj   rk   rl   )r$   r¡   ZS3FileSystemro   )r™   r1   Ús3_resourcer&   Ús3sor  Zs3Úkwr    r    r!   Útest_s3_roundtrip_explicit_fs  s    
ûz0TestParquetPyArrow.test_s3_roundtrip_explicit_fsc                 C   s   d|i}t ||d||d� d S )NÚstorage_optionsz s3://pandas-test/pyarrow.parquetr  ©ro   )r™   r1   r  r&   r  r    r    r!   Útest_s3_roundtrip+  s    ûz$TestParquetPyArrow.test_s3_roundtripr  Úpartition_colr-   c              
   C   sR   |  ¡ }|r*trdnd}||  |¡||< t|||dd|i|d |dœddd� d S )	NZint32r   zs3://pandas-test/parquet_dirr!  )Úpartition_colsr]   r!  Tr)   )ri   rj   rk   rl   ra   re   )rË   r   rY   ro   )r™   r1   r  r&   r$  r  Zexpected_dfZpartition_col_typer    r    r!   Útest_s3_roundtrip_for_dir7  s&    ÿýôz,TestParquetPyArrow.test_s3_roundtrip_for_dirr   c                 C   s(   t ƒ }| |¡ t|ƒ}t ||¡ d S r   )r   r   r   rc   rd   )r™   r1   ÚbufferZdf_from_bufr    r    r!   Útest_read_file_like_obj_supportb  s    
z2TestParquetPyArrow.test_read_file_like_obj_supportc              	   C   sb   |  dd¡ |  dd¡ tjtdd�� tdƒ W 5 Q R X tjtdd�� | d¡ W 5 Q R X d S )NÚHOMEZTestingUserÚUSERPROFILEz.*TestingUser.*r{   z~/file.parquet)Zsetenvr$   r}   ÚOSErrorr   r   )r™   r1   Zmonkeypatchr    r    r!   Útest_expand_useri  s    z#TestParquetPyArrow.test_expand_userc              	   C   sR   ddg}|}t  ¡ �4}|j||d d� t||ƒ t|ƒj|jksDt‚W 5 Q R X d S )NrV   rT   ©r%  r]   ©rc   Úensure_clean_dirr   rz   r   Úshapert   )r™   r&   r[   r%  rF   rj   r    r    r!   Útest_partition_cols_supportedr  s    

z0TestParquetPyArrow.test_partition_cols_supportedc              	   C   sT   d}|g}|}t  ¡ �4}|j||d d� t||ƒ t|ƒj|jksFt‚W 5 Q R X d S )NrV   r-  r.  )r™   r&   r[   r%  Úpartition_cols_listrF   rj   r    r    r!   Útest_partition_cols_string{  s    

z-TestParquetPyArrow.test_partition_cols_stringc           	   	   C   sP   d}|g}|}t  ¡ �0}||ƒ}|j||d� t|ƒj|jksBt‚W 5 Q R X d S )Nr.   )r%  )rc   r/  r   r   r0  rt   )	r™   r&   r1   r  r%  r2  rF   r  rj   r    r    r!   Útest_partition_cols_pathlib…  s    
z.TestParquetPyArrow.test_partition_cols_pathlibc                 C   s   t  ¡ }t||ƒ d S r   )r/   r0   ro   r  r    r    r!   Útest_empty_dataframe’  s    z'TestParquetPyArrow.test_empty_dataframec                 C   sV   dd l }t dddgi¡}| |jd| ¡ d�g¡}| t¡}t||d|i|d� d S )Nr   Úxr)   )Útyperw   rÓ   )	r   r/   r0   rw   ÚfieldZbool_rY   rV   ro   )r™   r&   r   rF   rw   Zout_dfr    r    r!   Útest_write_with_schema—  s
    
z)TestParquetPyArrow.test_write_with_schemac                 C   sz   t  t jdddgdd�t jdddgdd�t jdd dgd	d�d
œ¡}t||ƒ t  dt jdddd gdd�i¡}t||ƒ d S )Nr)   r*   r+   ré   r7   ZUInt32r<   rH   rQ   )r<   r=   rH   )r/   r0   r©   ro   r  r    r    r!   Ú test_additional_extension_arraysŸ  s    ýÿ
 z3TestParquetPyArrow.test_additional_extension_arraysz1.0.0)Úmin_versionc              	   C   sV   t  dt jdd dgdd�i¡}t  d|¡�" t||| d|› d�¡d� W 5 Q R X d S )	Nr<   rH   zstring[pyarrow]r7   Ústring_storagezstring[ú]©ri   )r/   r0   r©   rƒ   ro   rY   )r™   r&   r<  rF   r    r    r!   Ú test_pyarrow_backed_string_array¯  s    z3TestParquetPyArrow.test_pyarrow_backed_string_arrayz2.0.0c                 C   sN   t  t j dddg¡t jdddd�t j t jdddd�¡d	œ¡}t||ƒ d S )
N)r   r)   )r)   r*   )r+   r3   z
2012-01-01r+   ÚD)r;   rÁ   r3   )rH   r>   r?   )r/   r0   ZIntervalIndexr¿   Úperiod_rangeZfrom_breaksrE   ro   r  r    r    r!   Útest_additional_extension_types¶  s    ÿüÿ
z2TestParquetPyArrow.test_additional_extension_typesc                 C   s>   t s
d}nd}t dtjdddd�i¡}t||d|id	� d S )
Nz2.6z2.0r<   z
2017-01-01Z1nr  ©rÁ   r;   Úversionr³   )r	   r/   r0   rE   ro   )r™   r&   ÚverrF   r    r    r!   Útest_timestamp_nanosecondsÆ  s
    z-TestParquetPyArrow.test_timestamp_nanosecondsc                 C   sV   t s(|jtjjkr(|j tjj	dd�¡ d|g }t
j|d|id�}t||dd� d S )Nzitemporary skip this test until it is properly resolved: https://github.com/pandas-dev/pandas/issues/37286r   é   Úindex_as_col©r·   r  F)rb   )r   ÚtzinforW   ÚtimezoneÚutcrî   rï   r$   r¦   rí   r/   r0   ro   )r™   r   r&   r\   ÚidxrF   r    r    r!   Útest_timezone_aware_indexÑ  s    ÿþÿÿ
z,TestParquetPyArrow.test_timezone_aware_indexc              	   C   s^   t  dttddƒƒi¡}t ¡ �$}| ||¡ t||dgdd�}W 5 Q R X t|ƒdksZt	‚d S )Nr<   r   r+   ©r<   z==r   F)ÚfiltersZuse_legacy_datasetr)   )
r/   r0   rA   rB   rc   rn   r   r   rs   rt   )r™   r&   rF   rj   r”   r    r    r!   Útest_filter_row_groupsê  s    
   ÿz)TestParquetPyArrow.test_filter_row_groupsc              	   C   s~   t jtj dd¡dddgd�}t ¡ �}| ||¡ t||ƒ}W 5 Q R X |rdt	|j
t jjjƒszt‚nt	|j
t jjjƒszt‚d S )Nr  r+   r-   r.   ÚCrÃ   )r/   r0   rC   rÉ   rÊ   rc   rn   r   r   rˆ   Z_mgrÚcoreZ	internalsZArrayManagerrt   ZBlockManager)r™   r&   Zusing_array_managerrF   rj   r”   r    r    r!   Útest_read_parquet_managerõ  s    
z,TestParquetPyArrow.test_read_parquet_managerN)(r£   r¤   r¥   rü   rý   rÿ   r  r  r
  r  r$   r¦   rí   r   rõ   ÚstrÚpathlibÚPathr  r  Ú
single_cpur   r#  ÚtdZ
skip_if_nor&  r(  r,  r1  r3  r4  r5  r9  r:  r?  rB  rF  rN  rQ  rT  r    r    r    r!   rö   µ  s`   ý

þþ"

	






rö   c                   @   sŒ   e Zd Zdd„ Zdd„ Zdd„ Zdd„ Zd	d
„ Zdd„ Ze	j
jdd„ ƒZdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ ZdS ) ÚTestParquetFastParquetc                 C   sF   |}t jdddd�}| d ¡}||d< t jddd�|d< t||ƒ d S )	Nr9   r+   z
US/Easternrø   rú   r  r:   Ú	timedelta)r/   rE   r¸   r  ro   )r™   r(   r[   rF   rû   r    r    r!   rü     s    
z!TestParquetFastParquet.test_basicc                 C   s<   t jt d¡ dd¡tdƒd� ¡ }d}|  ||t|¡ d S )Nr   r3   r+   r  rÃ   z9Cannot create parquet dataset with duplicate column namesr  ©r™   r(   rF   r   r    r    r!   r    s    $z-TestParquetFastParquet.test_duplicate_columnsc                 C   sB   t  ddd dgi¡}t jddtjdgidd�}t|||dd� d S )	Nr<   TFrå   g        r  r7   )ri   rb   )r/   r0   rC   rX   ro   ©r™   r(   rF   ri   r    r    r!   Útest_bool_with_none  s    z*TestParquetFastParquet.test_bool_with_nonec                 C   sV   t  dt jdddd�i¡}|  ||td ¡ t  ddddgi¡}d}|  ||t|¡ d S )	Nr<   Z2013ÚMr+   rC  r)   rO   z"Can't infer object conversion type)r/   r0   rA  rœ   r~   r\  r    r    r!   r
    s
    z'TestParquetFastParquet.test_unsupportedc                 C   s&   t  dt  tdƒ¡i¡}t||ƒ d S )Nr<   r2   )r/   r0   r  rA   ro   )r™   r(   rF   r    r    r!   r  &  s    z'TestParquetFastParquet.test_categoricalc              	   C   sf   dt tddƒƒi}t |¡}t ¡ �(}|j||d dd� t||dgd�}W 5 Q R X t|ƒdksbt	‚d S )Nr<   r   r+   r)   )r]   Zrow_group_offsetsrO  )rP  )
rA   rB   r/   r0   rc   rn   r   r   rs   rt   )r™   r(   r>   rF   rj   r”   r    r    r!   rQ  *  s    

z-TestParquetFastParquet.test_filter_row_groupsc                 C   s    t ||dd|id |dœd� d S )Nz$s3://pandas-test/fastparquet.parquetr!  )r]   r!  r  r"  )r™   r1   r  r(   r  r    r    r!   r#  2  s    ûz(TestParquetFastParquet.test_s3_roundtripc              	   C   sl   ddg}|}t  ¡ �N}|j|d|d d� tj |¡s8t‚dd l}| |d¡j	}t
|ƒdks^t‚W 5 Q R X d S )NrV   rT   r   ©r"   r%  r]   r   Fr*   ©rc   r/  r   r  rj   Úexistsrt   r   ZParquetFileZcatsrs   ©r™   r(   r[   r%  rF   rj   r   Zactual_partition_colsr    r    r!   r1  =  s    
üz4TestParquetFastParquet.test_partition_cols_supportedc              	   C   sh   d}|}t  ¡ �N}|j|d|d d� tj |¡s4t‚dd l}| |d¡j	}t
|ƒdksZt‚W 5 Q R X d S )NrV   r   r`  r   Fr)   ra  rc  r    r    r!   r3  N  s    
üz1TestParquetFastParquet.test_partition_cols_stringc              	   C   sl   ddg}|}t  ¡ �N}|j|dd |d� tj |¡s8t‚dd l}| |d¡j	}t
|ƒdks^t‚W 5 Q R X d S )NrV   rT   r   )r"   r]   Úpartition_onr   Fr*   ra  rc  r    r    r!   Útest_partition_on_supported_  s    
üz2TestParquetFastParquet.test_partition_on_supportedc              
   C   sV   ddg}|}d}t jt|d��. t ¡ �}|j|dd ||d� W 5 Q R X W 5 Q R X d S )NrV   rT   zYCannot use both partition_on and partition_cols. Use partition_cols for partitioning datar{   r   )r"   r]   rd  r%  )r$   r}   r~   rc   r/  r   )r™   r(   r[   r%  rF   r   rj   r    r    r!   Ú3test_error_on_using_partition_cols_and_partition_onp  s    ÿ
ûzJTestParquetFastParquet.test_error_on_using_partition_cols_and_partition_onc                 C   s*   t  ¡ }| ¡ }d|j_t|||d� d S )Nr·   r>  ©r/   r0   rË   r·   r¹   ro   r]  r    r    r!   r5  ‚  s    z+TestParquetFastParquet.test_empty_dataframec                 C   s>   d|g }t j|d|id�}| ¡ }d|j_t|||d� d S )NrG  rH  rI  r·   r>  rg  )r™   r(   r\   rM  rF   ri   r    r    r!   rN  ‰  s
    
z0TestParquetFastParquet.test_timezone_aware_indexc              
   C   s\   t  dddgi¡}t ¡ �8}| |¡ tjtdd�� t|ddd� W 5 Q R X W 5 Q R X d S )	Nr<   r)   r*   z!not supported for the fastparquetr{   r   Trç   )	r/   r0   rc   rn   r   r$   r}   r~   r   )r™   r(   rF   rj   r    r    r!   Ú&test_use_nullable_dtypes_not_supported’  s
    

z=TestParquetFastParquet.test_use_nullable_dtypes_not_supportedc              
   C   sb   t  d¡�N}t |¡ d¡ tjtdd�� t|dd� W 5 Q R X t |¡j	dd� W 5 Q R X d S )	Nztest.parquets   breakitÚ r{   r   r�   F)Ú
missing_ok)
rc   rn   rV  rW  Úwrite_bytesr$   r}   Ú	Exceptionr   Úunlink)r™   rj   r    r    r!   Ú$test_close_file_handle_on_read_errorš  s
    z;TestParquetFastParquet.test_close_file_handle_on_read_errorN)r£   r¤   r¥   rü   r  r^  r
  r  rQ  r$   r¦   rX  r#  r1  r3  re  rf  r5  rN  rh  rn  r    r    r    r!   rZ    s    	

	rZ  )	NNNNNTFTr*   )MÚ__doc__rW   Úior   r  rV  Úwarningsr   r   ÚnumpyrC   r$   Zpandas._configr   Zpandas.compatr   Zpandas.compat.pyarrowr   r   r	   r
   Zpandas.util._test_decoratorsÚutilZ_test_decoratorsrY  Zpandasr/   Zpandas._testingZ_testingrc   Zpandas.util.versionr   Zpandas.io.parquetr   r   r   r   r   r   r#   r�   ÚDeprecationWarningÚFutureWarningr   r'   r¦   Z
pytestmarkZfixturer   Zskipifr"   r&   r(   r1   rG   r[   ÚnowrK  rL  ÚminÚmaxÚstrptimer\   ro   rz   r€   r„   r…   r‡   r‰   rŽ   r•   r–   r—   r¨   rö   rZ  r    r    r    r!   Ú<module>   sÄ   

ÿþþ ÿþøÿ





ùÿ
         ö
A+   $
  N