U
    ¯Ê¹j  ã                   @   s^   d Z ddlZddlm  mZ ddlmZ ddlmZ G dd„ dejƒZ	G dd	„ d	ejƒZ
dS )
a'   Split Attention Conv2d (for ResNeSt Models)

Paper: `ResNeSt: Split-Attention Networks` - /https://arxiv.org/abs/2004.08955

Adapted from original PyTorch impl at https://github.com/zhanghang1989/ResNeSt

Modified for torchscript compat, performance, and consistency with timm by Ross Wightman
é    N)Únné   )Úmake_divisiblec                       s$   e Zd Z‡ fdd„Zdd„ Z‡  ZS )ÚRadixSoftmaxc                    s   t t| ƒ ¡  || _|| _d S )N)Úsuperr   Ú__init__ÚradixÚcardinality)Úselfr   r	   ©Ú	__class__© úU/var/www/html/bwave/python/venv/lib/python3.8/site-packages/timm/layers/split_attn.pyr      s    zRadixSoftmax.__init__c                 C   sZ   |  d¡}| jdkrL| || j| jd¡ dd¡}tj|dd�}| |d¡}n
t 	|¡}|S )Nr   r   éÿÿÿÿé   ©Údim)
Úsizer   Úviewr	   Z	transposeÚFZsoftmaxÚreshapeÚtorchZsigmoid)r
   ÚxÚbatchr   r   r   Úforward   s    


zRadixSoftmax.forward)Ú__name__Ú
__module__Ú__qualname__r   r   Ú__classcell__r   r   r   r   r      s   r   c                       sH   e Zd ZdZdddddddddddejddf‡ fd	d
„	Zdd„ Z‡  ZS )Ú	SplitAttnz Split-Attention (aka Splat)
    Né   r   Fr   g      Ð?é   c                    s  t t| ƒ ¡  |p|}|	| _||	 }|d krDt||	 |
 d|d�}n||	 }|d kr\|d n|}tj||||||f||	 |dœ|—Ž| _|r”||ƒnt ¡ | _	|d k	r¬|ƒ nt ¡ | _
|dd�| _tj||d|d�| _|râ||ƒnt ¡ | _|dd�| _tj||d|d�| _t|	|ƒ| _d S )	Né    )Z	min_valueZdivisorr   )ÚgroupsÚbiasT)Zinplacer   )r#   )r   r   r   r   r   r   ZConv2dÚconvZIdentityÚbn0ÚdropÚact0Úfc1Úbn1Úact1Úfc2r   Úrsoftmax)r
   Zin_channelsZout_channelsZkernel_sizeZstrideÚpaddingZdilationr#   r$   r   Zrd_ratioZrd_channelsZ
rd_divisorZ	act_layerZ
norm_layerZ
drop_layerÚkwargsZmid_chsZattn_chsr   r   r   r   $   s:         ÿ þþzSplitAttn.__init__c           	      C   sþ   |   |¡}|  |¡}|  |¡}|  |¡}|j\}}}}| jdkrj| || j|| j ||f¡}|jdd�}n|}|jddd�}|  	|¡}|  
|¡}|  |¡}|  |¡}|  |¡ |ddd¡}| jdkrî|| || j|| j ddf¡ jdd�}n|| }| ¡ S )Nr   r   )r   r    T)Zkeepdimr   )r%   r&   r'   r(   Úshaper   r   ÚsumZmeanr)   r*   r+   r,   r-   r   Ú
contiguous)	r
   r   ÚBÚRCÚHÚWZx_gapZx_attnÚoutr   r   r   r   =   s&    









*zSplitAttn.forward)	r   r   r   Ú__doc__r   ZReLUr   r   r   r   r   r   r   r   !   s           þr   )r8   r   Ztorch.nn.functionalr   Z
functionalr   Zhelpersr   ÚModuler   r   r   r   r   r   Ú<module>   s   