U
    ¯Ê¹j3B  ã                   @   sú  d Z ddlZddlZddlZddlZddlZddlZddlZddlm	Z	 ddl
mZ ddlmZ ddlmZmZmZmZmZmZ ddlZddlmZ ddlZddlmZ ddlmZmZmZ z@ddl Z!dd	l"m#Z#m$Z$ dd
l%m&Z& ddl'm(Z(m)Z)m*Z*m+Z+ W n e,k
�r   dZ!dZ&Y nX ddl-m.Z. ddl/m0Z0 ddl1m2Z2 e 3e4¡Z5e6ej7 8dd¡ƒZ9e6ej7 8dd¡ƒZ:d,dd„Z;e	G dd„ dƒƒZ<e=edœdd„Z>dd„ Z?d-d"d#„Z@d$d%„ ZAe!dk	�rÒG d&d'„ d'e!jBƒZCndZCG d(d)„ d)eƒZDG d*d+„ d+e0ƒZEdS ).zR Dataset reader for webdataset

Hacked together by / Copyright 2022 Ross Wightman
é    N)Ú	dataclass)Úpartial)Úislice)ÚAnyÚCallableÚDictÚListÚOptionalÚTuple)ÚImage)ÚDatasetÚIterableDatasetÚget_worker_info)Ú_shuffleÚgetfirst)Úexpand_urls)Úbase_plus_extÚ
url_openerÚtar_file_expanderÚvalid_sampleé   )Úload_class_map)ÚReader)ÚSharedCountZWDS_SHUFFLE_SIZEi    ZWDS_INITIAL_SIZEi   ©z
_info.jsonz	info.jsonc           	      C   sÂ   t |tƒr|f}g }d}|D ]ˆ}tj | |¡}zJ| |¡ t |¡�&}| d¡r\t	 
|¡}n
t |¡}W 5 Q R X |W   S  tk
r¢ } zt|ƒ}W 5 d }~X Y qX qt d|› d|› d�¡ i S )NÚ z.jsonzDataset info file not found at z	. Error: z.. Falling back to provided split and size arg.)Ú
isinstanceÚstrÚosÚpathÚjoinÚappendÚwdsZgopenÚendswithÚjsonÚloadÚyamlZ	safe_loadÚ	ExceptionÚ_loggerÚwarning)	ÚrootÚnamesZtriedZerr_strÚnÚ	full_pathÚfZ	info_dictÚe© r0   ú[/var/www/html/bwave/python/venv/lib/python3.8/site-packages/timm/data/readers/reader_wds.pyÚ
_load_info*   s&    



ÿr2   c                   @   sJ   e Zd ZU eed< ee ed< dZee ed< dZeed< dZ	eed< dS )	Ú	SplitInfoÚnum_samplesÚ	filenamesr0   Úshard_lengthsr   Ú	alt_labelÚnameN)
Ú__name__Ú
__module__Ú__qualname__ÚintÚ__annotations__r
   r   r6   r7   r8   r0   r0   r0   r1   r3   B   s
   
r3   )ÚsplitÚinfoc           	         sz  dd„ }d| ksd| k�r$|   d¡} d}d}t| ƒdkrDt| d ƒ}| d } d	| kr�|   d
d¡}t|ƒd }|r�d|kr�|| |d kr�|| }t| ƒ}|�r|d | }|�s
dd„ t|d |d ƒD ƒ‰ t‡ fdd„|D ƒƒ}tˆ  ¡ ƒ|d< tˆ  ¡ ƒ|d< ||d< ||ƒ}nt	|||d�}nRd|k�s<| |d k�r^t
d| › d| di ¡ ¡ › d�ƒ‚| } |d |  }||ƒ}|S )Nc                 S   s2   t | d t| d ƒt| d ƒ|  dd¡| d d�S )Nr4   r5   r6   r7   r   r8   )r4   r5   r6   r7   r8   )r3   ÚtupleÚget)Z	dict_infor0   r0   r1   Ú_info_convertL   s    


ûz(_parse_split_info.<locals>._info_convertÚtarz..ú|r   r   r   z::ú-é   Zsplitsc                 S   s   i | ]\}}||“qS r0   r0   )Ú.0r.   Úcr0   r0   r1   Ú
<dictcomp>h   s      z%_parse_split_info.<locals>.<dictcomp>r5   r6   c                 3   s   | ]}ˆ | V  qd S ©Nr0   ©rG   r.   ©Z_fcr0   r1   Ú	<genexpr>i   s     z$_parse_split_info.<locals>.<genexpr>r4   )r8   r4   r5   zsplit z not found in info (ú))r>   Úlenr<   r   ÚzipÚsumr@   ÚkeysÚvaluesr3   ÚRuntimeErrorrA   )	r>   r?   rB   r4   Z
split_nameZsplit_partsZ	split_idxZsplit_filenamesÚ
split_infor0   rL   r1   Ú_parse_split_infoK   sD    	

ý"rV   c                 C   s(   t  dt| ƒ› d�¡ t| tƒr$| ‚dS )zRCall in an exception handler to ignore exceptions, isssue a warning, and continue.zHandling webdataset error (z). Ignoring.T)r(   r)   Úreprr   Ú	TypeError)Zexnr0   r0   r1   Úlog_and_continue~   s    
rY   ÚjpgÚRGBÚclsr   c           
   	   C   s’   |r,t  | d ¡}t|| ƒ}|dk r8dS nt| | ƒ}t| |ƒ}t |¡�}t |¡}| ¡  W 5 Q R X |rx| 	|¡}t
|||  dd¡d�}	|	S )z¡ Custom sample decode
    * decode and convert PIL Image
    * cls byte string label to int
    * pass through JSON byte string (if it exists) without parse
    r$   r   N)rZ   r\   r$   )r$   Úloadsr<   r   ÚioÚBytesIOr   Úopenr%   ÚconvertÚdictrA   )
ÚsampleÚ	image_keyÚ
image_modeÚ
target_keyr7   ÚmetaZclass_labelZimgÚbÚdecodedr0   r0   r1   Ú_decode‡   s    


rj   c                  C   s   t ƒ } | dk	r| jS tj ¡ S )z'get dataloader worker seed from pytorchN)r   Úseedr"   ÚutilsÚpytorch_worker_seed)Úworker_infor0   r0   r1   rm   «   s    rm   c                   @   s   e Zd Zd
dd„Zdd„ Zd	S )Údetshuffle2éè  éd   r   éÿÿÿÿc                 C   s   || _ || _|| _|| _d S rJ   )ÚbufsizeÚinitialrk   Úepoch)Úselfrs   rt   rk   ru   r0   r0   r1   Ú__init__¹   s    zdetshuffle2.__init__c                 C   sf   t | jtƒr| jj}n|  jd7  _| j}| jdk r@tƒ | }n
| j| }t |¡}t|| j	| j
|ƒS )Nr   r   )r   ru   r   Úvaluerk   rm   ÚrandomÚRandomr   rs   rt   )rv   Úsrcru   rk   Úrngr0   r0   r1   ÚrunÅ   s    



zdetshuffle2.runN)rp   rq   r   rr   )r9   r:   r;   rw   r}   r0   r0   r0   r1   ro   ¸   s       û
ro   c                       s4   e Zd ZdZejdddf‡ fdd„	Zdd„ Z‡  ZS )	ÚResampledShards2z,An iterable dataset yielding a list of urls.NTrr   c                    sb   t ƒ  ¡  tj |¡}|| _t| jd tƒs0t‚|| _	t
 ¡ | _|dkrLtn|| _|| _|| _dS )z‹Sample shards from the shard list with replacement.

        :param urls: a list of URLs as a Python list or brace notation string
        r   N)Úsuperrw   r"   Z
shardlistsr   Úurlsr   r   ÚAssertionErrorÚnshardsry   rz   r|   rm   Úworker_seedÚdeterministicru   )rv   r€   r‚   rƒ   r„   ru   ©Ú	__class__r0   r1   rw   Ý   s    

zResampledShards2.__init__c                 c   s‚   t | jtƒr| jj}n|  jd7  _| j}| jrDt |  ¡ | ¡| _t	| j
ƒD ].}| j dt| jƒd ¡}t| j| d�V  qNdS )z#Return an iterator over the shards.r   r   )ÚurlN)r   ru   r   rx   r„   ry   rz   rƒ   r|   Úranger‚   ÚrandintrO   r€   rb   )rv   ru   Ú_Úindexr0   r0   r1   Ú__iter__ó   s    
zResampledShards2.__iter__)	r9   r:   r;   Ú__doc__ÚsysÚmaxsizerw   rŒ   Ú__classcell__r0   r0   r…   r1   r~   Ú   s   úr~   c                       s®   e Zd Zd"eee eeee eeeee eeeeeee ee dœ‡ fdd„Zdd„ Z	d#ee dœdd„Z
dd„ Zdd„ Zdd„ Zdd„ Zdd„ Zd$dd„Zd%d d!„Z‡  ZS )&Ú	ReaderWdsNÚtrainFr   r   é*   újpg;png;webpr[   r\   r   Úfilename)r*   r8   r>   Úis_trainingr4   Ú
batch_sizeÚrepeatsrk   Ú	class_mapÚ	input_keyÚinput_img_moderf   Útarget_img_modeÚfilename_keyÚsample_shuffle_sizeÚsmaple_initial_sizec                    sR  t ƒ  ¡  td krtdƒ‚|| _|| _|| _|| _|| _d| _	|pDt
| _|pNt| _|
| _|| _|| _|| _d| _t| jƒ| _t|| jƒ| _|d k	rš|| _n
| jj| _| js²tdƒ‚d| _|	rÎt|	ƒ| _d| _ni | _d| _d| _t ¡ �rt ¡ �rt  ¡ dk�rt !¡ | _t  ¡ | _d | _"d| _#|| _$d| _%d| _&d| _'d| _(t)ƒ | _*d | _+d S )	NzcPlease install webdataset 0.2.x package `pip install git+https://github.com/webdataset/webdataset`.iô  z.JPEGz4Invalid split definition, num_samples not specified.FTr   r   ),r   rw   r"   rT   r*   r–   r—   r˜   Úcommon_seedÚshard_shuffle_sizeÚSAMPLE_SHUFFLE_SIZErž   ÚSAMPLE_INITIAL_SIZEÚsample_initial_sizerš   r›   rf   r�   Úkey_extr2   r?   rV   rU   r4   Úremap_classr   Úclass_to_idxÚ	dist_rankÚdist_num_replicasÚdistZis_availableZis_initializedZget_world_sizeZget_rankrn   Ú	worker_idrƒ   Únum_workersÚglobal_worker_idÚglobal_num_workersZ
init_countr   Úepoch_countÚds)rv   r*   r8   r>   r–   r4   r—   r˜   rk   r™   rš   r›   rf   rœ   r�   rž   rŸ   r…   r0   r1   rw     sX    
ÿ



"

zReaderWds.__init__c                 C   s   || j _d S rJ   )r¯   rx   )rv   Úcountr0   r0   r1   Ú	set_epochQ  s    zReaderWds.set_epoch)r¬   c                 C   s.   | j d k	rd S |d k	r*|| _| j| j | _d S rJ   )r°   r¬   r©   r®   )rv   r¬   r0   r0   r1   Úset_loader_cfgT  s
    
zReaderWds.set_loader_cfgc                    s.  ˆ j dkr^tjj ¡ }|dk	r<|ˆ _ |jˆ _|jˆ _|j	ˆ _	ˆ j
ˆ j	 ˆ _ˆ jˆ j	 ˆ j ˆ _‡ fdd„ˆ jjD ƒ}t |¡g}ˆ jrÌ| tˆ jˆ jˆ jd�ˆ jtjtd�tjˆ jˆ jt ˆ j¡d�g¡ n| ˆ jtjtd�g¡ | tj t!t"ˆ j#ˆ j$ˆ jj%d�td�tj&ˆ j#ˆ j'd�g¡ tj(|Ž ˆ _)dS )	z8 Lazily initialize worker (in worker processes)
        Nc                    s   g | ]}t j ˆ j|¡‘qS r0   )r   r   r    r*   rK   ©rv   r0   r1   Ú
<listcomp>l  s     z(ReaderWds._lazy_init.<locals>.<listcomp>)rk   ru   )Úhandler)rs   rt   r|   )rd   re   r7   )ÚimageÚtarget)*rn   Útorchrl   Údatar   Úidr«   rk   rƒ   r¬   r©   r®   r¨   r­   rU   r5   r"   ZSimpleShardListr–   Úextendro   r¡   r    r¯   Ú_split_by_node_and_workerZtarfile_to_samplesrY   Úshufflerž   r¤   ry   rz   Úmapr   rj   rš   r›   r7   Úrenamerf   ZDataPipeliner°   )rv   rn   Zabs_shard_filenamesZpipeliner0   r´   r1   Ú
_lazy_init^  sV    
ý

ý÷
ýüù	özReaderWds._lazy_initc                 c   s>   | j dkr*t|| jd | j ƒD ]
}|V  qn|D ]
}|V  q.d S ©Nr   )r®   r   r­   )rv   r{   Úsr0   r0   r1   r½   “  s
    

z#ReaderWds._split_by_node_and_workerc                 C   sR   | j t| j| jƒ }| js$| jdkr.t |¡}| jrJt || j ¡| j }t|ƒS rÂ   )	r4   Úmaxr®   r©   r–   ÚmathÚceilr—   r<   )rv   Únum_worker_samplesr0   r0   r1   Ú_num_samples_per_worker›  s    
z!ReaderWds._num_samples_per_workerc                 c   s~   | j d kr|  ¡  |  ¡ }| js*| jdkr8| j  |¡}n| j }d}|D ]2}|d }| jrb| j| }|d |fV  |d7 }qFd S )Nr   r   r¸   r·   )r°   rÁ   rÈ   r–   r©   Z
with_epochr¦   r§   )rv   rÇ   r°   Úirc   r¸   r0   r0   r1   rŒ   £  s    

zReaderWds.__iter__c                 C   s   |   ¡ | j }|S rJ   )rÈ   r¬   )rv   r4   r0   r0   r1   Ú__len__»  s    zReaderWds.__len__c                 C   s   dst dƒ‚d S )NFzNot supported)r�   )rv   r‹   ÚbasenameÚabsoluter0   r0   r1   Ú	_filename¿  s    zReaderWds._filenamec                 C   s|   | j dkr|  ¡  g }| j D ]Z}| j|kr6|| j }n$d|krN|d | j }ndsZtdƒ‚| |¡ t|ƒ| jkr qxq|S )z0 Return all filenames in dataset, overrides baseNZ__key__FzNo supported name field present)r°   rÁ   r�   r¥   r�   r!   rO   r4   )rv   rË   rÌ   r+   rc   r8   r0   r0   r1   r5   Â  s    



zReaderWds.filenames)Nr’   FNr   r   r“   Nr”   r[   r\   r   r•   NN)N)FF)FF)r9   r:   r;   r   r	   Úboolr<   rb   rw   r²   r³   rÁ   r½   rÈ   rŒ   rÊ   rÍ   r5   r�   r0   r0   r…   r1   r‘     sZ                  ïïJ þþ
5
r‘   )r   )rZ   r[   r\   r   )Fr�   r^   r$   ÚloggingrÅ   r   ry   rŽ   Údataclassesr   Ú	functoolsr   Ú	itertoolsr   Útypingr   r   r   r   r	   r
   r¹   Ztorch.distributedÚdistributedrª   r&   ZPILr   Ztorch.utils.datar   r   r   Z
webdatasetr"   Zwebdataset.filtersr   r   Zwebdataset.shardlistsr   Zwebdataset.tariteratorsr   r   r   r   ÚImportErrorr™   r   Úreaderr   Zshared_countr   Ú	getLoggerr9   r(   r<   ÚenvironrA   r¢   r£   r2   r3   r   rV   rY   rj   rm   ZPipelineStagero   r~   r‘   r0   r0   r0   r1   Ú<module>   s\    


3    û
$

,