U
    `»{j5%  ã                   @   s2  d Z ddlmZ ddlZddlZddlZddlZddlZddl	m
Z
mZ ddlmZ ej d¡Zedd„ ƒZedd	„ ƒZeej d
ddg¡ej ddddg¡dd„ ƒƒƒZedd„ ƒZedd„ ƒZeej ddi e
ddgiƒfdddie
ddgiƒfdddgie
dd d!giƒfd"dgd#d$œe
ddgiƒfd"dgd%d$œe
dejdgiƒfg¡d&d'„ ƒƒZed(d)„ ƒZeej d*d+d,d-g¡d.d/„ ƒƒZeej d0d#d%g¡d1d2„ ƒƒZed3d4„ ƒZej dd5dd6d7d8g¡d9d:„ ƒZeej dd5ddd;d<g¡d=d>„ ƒƒZed?d@„ ƒZ edAdB„ ƒZ!ej dC¡ej dDdEdFg¡dGdH„ ƒƒZ"dS )IzZ
Tests encoding functionality during parsing
for all of the parsers defined in parsers.py
é    )ÚBytesION)Ú	DataFrameÚread_csvZpyarrow_skipc                 C   sL   d}| }t d |¡ƒ}|j|d|d�}tddggddgd	�}t ||¡ d S )
NÚcp1255u   ×©×œ×•×�:1234
562:123ú:)ÚsepÚencodingi2  é{   u   ×©×œ×•×�Ú1234©Úcolumns©r   Úencoder   r   ÚtmÚassert_frame_equal)Úall_parsersr   ÚparserÚdataÚresultÚexpected© r   úc/var/www/html/bwave/python/venv/lib/python3.8/site-packages/pandas/tests/io/parser/test_encoding.pyÚtest_bytes_io_input   s    r   c                 C   s@   | }t d ¡ ƒ}|j|ddd d�}tddggƒ}t ||¡ d S )Nu   Å�aski, Jan;1ú;úutf-8)r   r   Úheaderu   Å�aski, Jané   r   )r   r   r   r   r   r   r   r   Útest_read_csv_unicode"   s
    r   r   ú,ú	r   úutf-16zutf-16lezutf-16bec              
   C   sØ   | }d  d|¡}dt ¡ › d�}|ddœ}d}t |¡�–}dd	lm} | |¡}	t|d
ƒ�}
|
 	|	¡ W 5 Q R X t
| |¡ƒ}|||d�}|j|fd|i|—Ž}|j|fd|i|—Ž}| ¡  t ||¡ W 5 Q R X d S )Nz)skip this
skip this too
A,B,C
1,2,3
4,5,6r   Ú__z__.csvé   )r   Zskiprowsr   r   )ÚTextIOWrapperÚwb©r   r   )ÚreplaceÚuuidÚuuid4r   Úensure_cleanÚior#   r   ÚopenÚwriter   r   Úcloser   )r   r   r   r   r   ÚpathÚkwargsÚutf8r#   Ú
bytes_dataÚfZbytes_bufferr   r   r   r   r   Útest_utf16_bom_skiprows,   s&     û

r3   c                 C   s6   t j |d¡}| }|j|ddd�}t|ƒdks2t‚d S )Nzutf16_ex.txtr    r   )r   r   é2   )Úosr.   Újoinr   ÚlenÚAssertionError)r   Úcsv_dir_pathr.   r   r   r   r   r   Útest_utf16_exampleO   s    r:   c                 C   sL   t j |d¡}| }|j|d dd�}| d¡}|d d }d}||ksHt‚d S )Núunicode_series.csvúlatin-1)r   r   r   r   i`  u$   Ã� kÃ¶ldum klaka (Cold Fever) (1994))r5   r.   r6   r   Z	set_indexr8   )r   r9   r.   r   r   Úgotr   r   r   r   Útest_unicode_encodingW   s    
r>   zdata,kwargs,expectedza
1Úar   z"a"
1Ú	quotecharú"zb
1ÚnamesÚbÚ1z
1T)rB   Zskip_blank_linesFc                    sD   | }d‰ d‰‡ ‡fdd„}|j ||ƒfdˆi|—Ž}t ||¡ d S )Nu   ï»¿r   c                    s   ˆ |    ˆ¡}t|ƒS )N)r   r   )Ú_dataZbom_data©Úbomr0   r   r   Ú_encode_data_with_bom~   s    z,test_utf8_bom.<locals>._encode_data_with_bomr   )r   r   r   )r   r   r/   r   r   rH   r   r   rF   r   Útest_utf8_bomd   s    rI   c                 C   sL   t dgdgdœƒ}| }| |¡}d |¡}|jt|ƒ|d�}t ||¡ d S )Ng333333@Útest)Zmb_numZ	multibytezmb_num,multibyte
4.8,testr%   )r   Úformatr   r   r   r   r   )r   Ú	utf_valueÚencoding_fmtr   r   r   r   r   r   r   r   Útest_read_csv_utf_aliases†   s    

rN   zfile_path,encoding))r*   r   Úcsvz	test1.csvr   ))r*   r   r   r;   r<   ))r*   r   r   zsauron.SHIFT_JIS.csvÚshiftjisc           
   	   C   sÐ   | }||Ž }|j ||d�}t||d��}|  |¡}|jr<t‚W 5 Q R X t ||¡ t|dd��}	|j |	|d�}|	jrxt‚W 5 Q R X t ||¡ t|ddd��}	|j |	|d�}|	jr¶t‚W 5 Q R X t ||¡ d S )Nr%   Úrb©Úmoder   )rS   Ú	buffering)r   r+   Úclosedr8   r   r   )
r   Ú	file_pathr   Údatapathr   Zfpathr   Úfar   Zfbr   r   r   Útest_binary_mode_file_buffers“   s    
rY   Úpass_encodingc           	   	   C   sr   | }|  |¡}tddgiƒ}tjd|dd��<}| d¡ | d¡ |j||rP|nd d�}t ||¡ W 5 Q R X d S )	NZfooÚbarzw+T)rS   r   Zreturn_filelikezfoo
barr   r%   )rK   r   r   r)   r,   Úseekr   r   )	r   rL   rM   rZ   r   r   r   r2   r   r   r   r   Útest_encoding_temp_file´   s    


r]   c              	   C   s~   | }d}d}d}t ||giƒ}t ¡ �N}| |› d|› � |¡¡ | d¡ |j||d�}t ||¡ |j	rpt
‚W 5 Q R X d S )Nz	shift-jisu	   ã�¦ã�™ã�¨u   ã�“ã‚€Ú
r   r%   )r   ÚtempfileÚNamedTemporaryFiler,   r   r\   r   r   r   rU   r8   )r   r   r   Útitler   r   r2   r   r   r   r   Útest_encoding_named_temp_fileÅ   s    

rb   r   z	utf-16-bez	utf-16-lezutf-32c                 C   sR   d}t | | ¡ƒ}t|d| d�}tddgddgdd	ggd
dgd�}t ||¡ d S )Nu   a	b
ï¼šfoo	0
bar	1
baz	2r   )Ú	delimiterr   u   ï¼šfoor   r[   r   Zbazr"   r?   rC   )r   r   r   )r   r   Zencoded_datar   r   r   r   r   Ú%test_parse_encoded_special_charactersÚ   s
    "rd   r   r<   c              	   C   sp   | }t ddddgddddgd	d
ddgdœƒ}t ¡ �&}|j|d|d� |j||dd�}W 5 Q R X t ||¡ d S )NZRaphaelZ	DonatellozMiguel AngelZLeonardoÚredÚpurpleZorangeÚblueZsaizbo staffZnunchunkZkatana)ÚnameÚmaskZweaponF)Úindexr   T)r   Ú
memory_map)r   r   r)   Úto_csvr   r   )r   r   r   r   ÚfileÚdfr   r   r   Útest_encoding_memory_mapè   s    


ýÿ
ro   c              	   C   sh   | }t dgd d�}d|jd< t d¡�*}|j|dddd	� |j|d
ddd�}W 5 Q R X t ||¡ d
S )zO
    Chunk splits a multibyte character with memory_map=True

    GH 43540
    Zaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaai   )r   u�   aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaÄ…iÿ  zbug-gh43540.csvFr   ©rj   r   r   NTÚc)r   rk   Úengine)r   Zilocr   r)   rl   r   r   )r   r   rn   ÚfnameÚdfrr   r   r   Ú test_chunk_splits_multibyte_charú   s    
ru   c              	   C   sÖ   g }d}d}d}t t|ƒt|ƒ|ƒD ]X}d dd„ t ||d ƒD ƒ¡d }z| d¡ W n tk
rp   Y q$Y nX | |¡ q$| }t|ƒ}t d	¡�,}	|j	|	d
d
dd� |j
|	ddddd�}
W 5 Q R X t ||
¡ dS )zg
    GH 43787

    Test correct handling of UTF-8 chars when memory_map=True and encoding is UTF-8
    é€   ú u   ð�‚€Ú c                 S   s   g | ]}t |ƒ‘qS r   )Úchr)Ú.0rq   r   r   r   Ú
<listcomp>  s     z,test_readcsv_memmap_utf8.<locals>.<listcomp>r^   r   zutf8test.csvFrp   NTrq   )r   rk   rr   r   )ÚrangeÚordr6   r   ÚUnicodeEncodeErrorÚappendr   r   r)   rl   r   r   )r   ÚlinesÚline_lengthZ
start_charZend_charÚlnumÚliner   rn   rs   rt   r   r   r   Útest_readcsv_memmap_utf8  s.    "
    ÿr„   Zpyarrow_xfailrS   zw+bzw+tc              	   C   sh   | }d}d|krd}t j|d��$}| |¡ | d¡ | |¡}W 5 Q R X tg dgd�}t ||¡ d S )Ns   abcdÚtÚabcdrR   r   r   )r_   ÚSpooledTemporaryFiler,   r\   r   r   r   r   )r   rS   r   ÚcontentÚhandlern   r   r   r   r   Útest_not_readable,  s    

rŠ   )#Ú__doc__r*   r   r5   r_   r'   ÚnumpyÚnpZpytestZpandasr   r   Zpandas._testingZ_testingr   ÚmarkZusefixturesZskip_pyarrowr   r   Zparametrizer3   r:   r>   ÚnanrI   rN   rY   r]   rb   rd   ro   ru   r„   rŠ   r   r   r   r   Ú<module>   s„   

	 


ýöþ
ýþ
 ÿ



