U
    ¯Ê¹j`  ã                   @   s„   d dl mZ d dlZd dlmZ d dlmZmZ d dlmZ dd„ ZG dd	„ d	ejƒZ	ej
jG d
d„ deƒƒZG dd„ dejƒZdS )é    )ÚOptionalN)Únn)r   ÚTensor)Ú_get_activation_fnc                 C   sØ   t | dƒr>t | dƒr>t ¡ | _| `| j}| j}t||d�| _n†t | dƒr|t | dƒr|t ¡ | _| `| j}| j}t||d�| _nHd|  	¡ ks”d|  	¡ kr´| `
| j}| j}t||d�| _
ntdƒ tdƒ t | d	ƒrÔd
| _| S )NÚglobal_poolÚfc)Únum_classesÚinitial_num_featuresÚ
classifierZRegNetZTResNetz;Model code-writing is not aligned currently with ml-decoderéÿÿÿÿÚ	drop_rater   )Úhasattrr   ZIdentityr   r   r   Únum_featuresÚ	MLDecoderr
   Z	_get_nameÚheadÚprintÚexitr   )Úmodelr   r   © r   úU/var/www/html/bwave/python/venv/lib/python3.8/site-packages/timm/layers/ml_decoder.pyÚadd_ml_decoder_head	   s,    


r   c                	       s\   e Zd Zdddœ‡ fdd	„Z‡ fd
d„Zdeeee ee ee ee edœdd„Z‡  ZS )ÚTransformerDecoderLayerOptimalé   é   çš™™™™™¹?Úreluçñhãˆµøä>N)Úreturnc                    sª   t t| ƒ ¡  tj||d�| _t |¡| _t |¡| _t |¡| _	t |¡| _
tj|||d�| _t ||¡| _t ||¡| _tj||d�| _tj||d�| _t|ƒ| _d S )N)Úeps)Údropout)Úsuperr   Ú__init__r   Z	LayerNormÚnorm1ZDropoutr   Údropout1Údropout2Údropout3ZMultiheadAttentionÚmultihead_attnÚLinearÚlinear1Úlinear2Únorm2Únorm3r   Ú
activation)ÚselfÚd_modelZnheadÚdim_feedforwardr   r,   Zlayer_norm_eps©Ú	__class__r   r   r!   $   s    z'TransformerDecoderLayerOptimal.__init__c                    s*   d|krt jjj|d< tt| ƒ |¡ d S )Nr,   )Útorchr   Ú
functionalr   r    r   Ú__setstate__)r-   Ústater0   r   r   r4   8   s    z+TransformerDecoderLayerOptimal.__setstate__)ÚtgtÚmemoryÚtgt_maskÚmemory_maskÚtgt_key_padding_maskÚmemory_key_padding_maskr   c              	   C   sz   ||   |¡ }|  |¡}|  |||¡d }||  |¡ }|  |¡}|  |  |  |  |¡¡¡¡}||  	|¡ }|  
|¡}|S )Nr   )r#   r"   r&   r$   r*   r)   r   r,   r(   r%   r+   )r-   r6   r7   r8   r9   r:   r;   Ztgt2r   r   r   Úforward=   s    


z&TransformerDecoderLayerOptimal.forward)r   r   r   r   r   )NNNN)	Ú__name__Ú
__module__Ú__qualname__r!   r4   r   r   r<   Ú__classcell__r   r   r0   r   r   #   s      ÿÿ    ý ýr   c                   @   s2   e Zd Zedœdd„Zejejejdœdd„ZdS )ÚGroupFC©Úembed_len_decoderc                 C   s
   || _ d S ©NrB   )r-   rC   r   r   r   r!   ]   s    zGroupFC.__init__)ÚhÚduplicate_poolingÚ
out_extrapc                 C   s^   t | jƒD ]N}|d d …|d d …f }||d d …d d …f }t ||¡|d d …|d d …f< q
d S rD   )ÚrangerC   r2   Úmatmul)r-   rE   rF   rG   ÚiZh_iZw_ir   r   r   Ú__call__`   s    zGroupFC.__call__N)r=   r>   r?   Úintr!   r2   r   rK   r   r   r   r   rA   [   s   rA   c                       s&   e Zd Zd‡ fdd„	Zdd„ Z‡  ZS )	r   r   é   r   c           
         s  t t| ƒ ¡  |dk rdn|}||kr*|}|dk r6dn|}t ||¡| _d}d}d}t|||d�}	tj|	|d�| _t 	||¡| _
| j
 d	¡ || _t|| d
 ƒ| _tj t ||| j¡¡| _tj t |¡¡| _tjj | j¡ tjj | jd¡ t|ƒ| _d S )Nr   éd   rM   r   é   r   )r.   r/   r   )Z
num_layersFg+‡ÙÎ÷ï?)r    r   r!   r   r'   Úembed_standartr   ZTransformerDecoderÚdecoderZ	EmbeddingÚquery_embedZrequires_grad_r   rL   Úduplicate_factorr2   Ú	Parameterr   rF   Úduplicate_pooling_biasÚinitZxavier_normal_Z	constant_rA   Úgroup_fc)
r-   r   Znum_of_groupsZdecoder_embeddingr	   rC   Zdecoder_dropoutZnum_layers_decoderr/   Zlayer_decoder0   r   r   r!   h   s2     ÿÿzMLDecoder.__init__c                 C   sì   t |jƒdkr"| d¡ dd¡}n|}|  |¡}tjjj|dd�}|jd }| j	j
}| d¡ d|d¡}|  || dd¡¡}| dd¡}tj|jd |jd | j|j|jd�}|  || j|¡ | d¡d d …d | j…f }	|	| j7 }	|	}
|
S )	Né   é   rO   T)Zinplacer   r   )ÚdeviceÚdtype)ÚlenÚshapeÚflattenZ	transposerP   r2   r   r3   r   rR   ÚweightZ	unsqueezeÚexpandrQ   ZzerosrS   rZ   r[   rW   rF   r   rU   )r-   ÚxZembedding_spatialZembedding_spatial_786ÚbsrR   r6   rE   rG   Zh_outZlogitsr   r   r   r<   ˆ   s     

&
zMLDecoder.forward)r   rM   r   )r=   r>   r?   r!   r<   r@   r   r   r0   r   r   g   s    r   )Útypingr   r2   r   r   Ztorch.nn.modules.transformerr   r   ÚModuler   ZjitÚscriptÚobjectrA   r   r   r   r   r   Ú<module>   s   8