U
    ¯Ê¹jZ   ã                
   @   sî   d Z ddlZddlZddlmZmZ ddlmZ ddlZddl	m
Z ddlmZ z$ddlZddlmZ ddlmZ W n. ek
r¢ Z zedƒ e‚W 5 dZ[X Y nX d	d
lmZ d	dlmZ d	dlmZ eej dd¡ƒZG dd„ deƒZ dS )z' Dataset reader for HF IterableDataset
é    N)ÚrepeatÚchain)ÚOptional)ÚImage)Úsplit_dataset_by_node)Ú	SplitInfozDPlease install Hugging Face datasets package `pip install datasets`.é   )Úload_class_map)ÚReader)ÚSharedCountZHFIDS_SHUFFLE_SIZEi   c                       s    e Zd Zdeee eeeeeeee eeeeee ee dœ‡ fdd„Zdd„ Z	d ee dœdd„Z
dd„ Zdd„ Zdd„ Zdd„ Zd!dd„Zd"dd„Z‡  ZS )#ÚReaderHfidsNÚtrainFr   r   é*   ÚimageÚRGBÚlabelÚ )ÚnameÚrootÚsplitÚis_trainingÚ
batch_sizeÚdownloadÚrepeatsÚseedÚ	class_mapÚ	input_keyÚinput_img_modeÚ
target_keyÚtarget_img_modeÚshuffle_sizeÚnum_samplesc                    sn  t ƒ  ¡  || _|| _|| _|| _|| _|| _|| _|p:t	| _
|
| _|| _|| _|| _tj||d�| _|rt| j ¡  d }| jjjr²|| jjjkr²t| jjj| tƒr²| jjj| }|r¾|| _n|rÒ|jrÒ|j| _ntdƒ‚d| _|	röt|	ƒ| _d| _ni | _d| _d| _t  ¡ �r>t !¡ �r>t "¡ dk�r>t #¡ | _t "¡ | _d | _$d| _%d| _&d| _'d| _(d | _)t*ƒ | _+d S )N)Ú	cache_dirz“Dataset length is unknown, please pass `num_samples` explicitely. The number of steps needs to be known in advance for the learning rate scheduler.FTr   r   ),ÚsuperÚ__init__r   r   r   r   r   r   Úcommon_seedÚSHUFFLE_SIZEr    r   r   r   r   ÚdatasetsZload_dataset_builderÚbuilderZdownload_and_prepareÚinfoZsplitsÚ
isinstancer   r!   Znum_examplesÚ
ValueErrorÚremap_classr	   Úclass_to_idxÚ	dist_rankÚdist_num_replicasÚdistZis_availableZis_initializedZget_world_sizeZget_rankÚworker_infoÚ	worker_idÚnum_workersÚglobal_worker_idÚglobal_num_workersÚdsr   Úepoch)Úselfr   r   r   r   r   r   r   r   r   r   r   r   r   r    r!   Z
split_info©Ú	__class__© ú]/var/www/html/bwave/python/venv/lib/python3.8/site-packages/timm/data/readers/reader_hfids.pyr$      sX    




ÿ
"

zReaderHfids.__init__c                 C   s   || j _d S ©N)r7   Úvalue)r8   Úcountr;   r;   r<   Ú	set_epochk   s    zReaderHfids.set_epoch)r3   c                 C   s.   | j d k	rd S |d k	r*|| _| j| j | _d S r=   )r6   r3   r/   r5   )r8   r3   r;   r;   r<   Úset_loader_cfgo   s
    
zReaderHfids.set_loader_cfgc                 C   s¼   | j dkrVtjj ¡ }|dk	r4|| _ |j| _|j| _| j| j | _	| j
| j | j | _| jr|| jj| jd�}|j| j	d�}n| jj| jd�}| jr¤|j| j| jd�}t|| j
| jd�| _dS )z8 Lazily initialize worker (in worker processes)
        N)r   )Z
num_shards)r   Úbuffer_size)ZrankZ
world_size)r1   ÚtorchÚutilsÚdataZget_worker_infoÚidr2   r3   r/   r5   r.   r4   r   r(   Z
as_datasetr   Zto_iterable_datasetZas_streaming_datasetr   Úshuffler%   r    r   r6   )r8   r1   Zdatasetr6   r;   r;   r<   Ú
_lazy_inity   s    
	zReaderHfids._lazy_initc                 C   sh   t d| jƒ| j t | j| jƒ }| js0| jdkr:t |¡}| jr`| jd k	r`t || j ¡| j }t	|ƒS )Nr   )
Úmaxr   r!   r5   r/   r   ÚmathÚceilr   Úint)r8   Znum_worker_samplesr;   r;   r<   Ú_num_samples_per_workerš   s    ÿ
z#ReaderHfids._num_samples_per_workerc                 c   sø   | j d kr|  ¡  | j  | jj¡ |  ¡ }d}| jrFt t	| j ƒ¡}n
t
| j ƒ}|D ]ž}|| j }| jr€|j| jkr€| | j¡}|| j }| jr¾t|tjƒs¤tdƒ‚|j| jkrÎ| | j¡}n| jrÎ| j| }||fV  |d7 }| jrT||krT qôqTd S )Nr   z7target_img_mode is specified but target is not an imager   )r6   rH   r@   r7   r>   rM   r   r   Úfrom_iterabler   Úiterr   r   ÚmodeÚconvertr   r   r*   r   ÚAssertionErrorr,   r-   )r8   Ztarget_sample_countZsample_countZds_iterÚsampleZ
input_dataZtarget_datar;   r;   r<   Ú__iter__£   s.    





zReaderHfids.__iter__c                 C   s   |   ¡ | j }|S r=   )rM   r3   )r8   r!   r;   r;   r<   Ú__len__¿   s    zReaderHfids.__len__c                 C   s   dst dƒ‚d S )NFzNot supported)rR   )r8   ÚindexÚbasenameÚabsoluter;   r;   r<   Ú	_filenameÃ   s    zReaderHfids._filenamec                 C   s„   | j dkr|  ¡  g }| j D ]b}d|kr2|d }nBd|krD|d }n0d|krV|d }nd|krh|d }ndsttdƒ‚| |¡ q|S )z0 Return all filenames in dataset, overrides baseNÚ	file_nameÚfilenamerF   Zimage_idFzNo supported name field present)r6   rH   rR   Úappend)r8   rW   rX   ÚnamesrS   r   r;   r;   r<   Ú	filenamesÆ   s    





zReaderHfids.filenames)Nr   Fr   Fr   r   Nr   r   r   r   NN)N)FF)FF)Ú__name__Ú
__module__Ú__qualname__Ústrr   ÚboolrL   Údictr$   r@   rA   rH   rM   rT   rU   rY   r^   Ú__classcell__r;   r;   r9   r<   r      sT                 ððM þþ
!	
r   )!Ú__doc__rJ   ÚosÚ	itertoolsr   r   Útypingr   rC   Ztorch.distributedÚdistributedr0   ZPILr   r'   Zdatasets.distributedr   Zdatasets.splitsr   ÚImportErrorÚeÚprintr   r	   Úreaderr
   Zshared_countr   rL   ÚenvironÚgetr&   r   r;   r;   r;   r<   Ú<module>   s&   