U
    ¯Ê¹j	!  ã                   @   s´   d Z ddlmZ ddlmZ ddlmZ ddlmZ G dd„ dej	ƒZ
G d	d
„ d
ej	ƒZeeejdd�ZG dd„ dej	ƒZG dd„ dej	ƒZG dd„ dej	ƒZG dd„ dej	ƒZdS )zl MLP module w/ dropout and configurable activation layer

Hacked together by / Copyright 2020 Ross Wightman
é    )Úpartial)Únné   )ÚGlobalResponseNorm)Ú	to_2tuplec                       s:   e Zd ZdZddejddddf‡ fdd„	Zdd	„ Z‡  ZS )
ÚMlpzG MLP as used in Vision Transformer, MLP-Mixer and related networks
    NTç        Fc	                    s°   t ƒ  ¡  |p|}|p|}t|ƒ}t|ƒ}	|r<ttjdd�ntj}
|
|||d d�| _|ƒ | _t 	|	d ¡| _
|d k	r~||ƒnt ¡ | _|
|||d d�| _t 	|	d ¡| _d S )Nr   ©Úkernel_sizer   ©Úbias)ÚsuperÚ__init__r   r   r   ÚConv2dÚLinearÚfc1ÚactÚDropoutÚdrop1ÚIdentityÚnormÚfc2Údrop2)ÚselfÚin_featuresÚhidden_featuresÚout_featuresÚ	act_layerÚ
norm_layerr   ÚdropÚuse_convÚ
drop_probsÚlinear_layer©Ú	__class__© úN/var/www/html/bwave/python/venv/lib/python3.8/site-packages/timm/layers/mlp.pyr      s    
zMlp.__init__c                 C   s@   |   |¡}|  |¡}|  |¡}|  |¡}|  |¡}|  |¡}|S ©N)r   r   r   r   r   r   ©r   Úxr%   r%   r&   Úforward)   s    





zMlp.forward©	Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   ZGELUr   r*   Ú__classcell__r%   r%   r#   r&   r      s   ÷r   c                       sD   e Zd ZdZddejdddddf‡ fdd„	Zdd	„ Zd
d„ Z‡  Z	S )ÚGluMlpzi MLP w/ GLU style gating
    See: https://arxiv.org/abs/1612.08083, https://arxiv.org/abs/2002.05202
    NTr   Fc
                    sÜ   t ƒ  ¡  |p|}|p|}|d dks*t‚t|ƒ}t|ƒ}
|rLttjdd�ntj}|rZdnd| _|	| _	||||d d�| _
|ƒ | _t |
d ¡| _|d k	r¦||d ƒnt ¡ | _||d ||d d�| _t |
d ¡| _d S )Né   r   r   r	   éÿÿÿÿr   )r   r   ÚAssertionErrorr   r   r   r   r   Ú	chunk_dimÚ	gate_lastr   r   r   r   r   r   r   r   )r   r   r   r   r   r   r   r   r    r6   r!   r"   r#   r%   r&   r   7   s    
zGluMlp.__init__c                 C   sJ   | j jjd d }tj | j j|d … ¡ tjj| j j|d … dd� d S )Nr   r2   ç�íµ ÷Æ°>©Zstd)r   r   Úshaper   ÚinitÚones_Únormal_Úweight)r   Zfc1_midr%   r%   r&   Úinit_weightsT   s    zGluMlp.init_weightsc                 C   sl   |   |¡}|jd| jd�\}}| jr2||  |¡ n|  |¡| }|  |¡}|  |¡}|  |¡}|  |¡}|S )Nr2   )Údim)	r   Úchunkr5   r6   r   r   r   r   r   )r   r)   Úx1Zx2r%   r%   r&   r*   Z   s    
"



zGluMlp.forward)
r,   r-   r.   r/   r   ZSigmoidr   r>   r*   r0   r%   r%   r#   r&   r1   3   s   ör1   F)r   r6   c                       s@   e Zd ZdZddejdddf‡ fdd„	Zdd„ Zd	d
„ Z‡  Z	S )ÚSwiGLUz¬ SwiGLU
    NOTE: GluMLP above can implement SwiGLU, but this impl has split fc1 and
    better matches some other common impl which makes mapping checkpoints simpler.
    NTr   c           	         s²   t ƒ  ¡  |p|}|p|}t|ƒ}t|ƒ}tj|||d d�| _tj|||d d�| _|ƒ | _t |d ¡| _	|d k	r~||ƒnt 
¡ | _tj|||d d�| _t |d ¡| _d S )Nr   r   r   )r   r   r   r   r   Úfc1_gÚfc1_xr   r   r   r   r   r   r   )	r   r   r   r   r   r   r   r   r!   r#   r%   r&   r   m   s    

zSwiGLU.__init__c                 C   s(   t j | jj¡ t jj| jjdd� d S )Nr7   r8   )r   r:   r;   rC   r   r<   r=   )r   r%   r%   r&   r>   …   s    zSwiGLU.init_weightsc                 C   sN   |   |¡}|  |¡}|  |¡| }|  |¡}|  |¡}|  |¡}|  |¡}|S r'   )rC   rD   r   r   r   r   r   )r   r)   Zx_gater%   r%   r&   r*   Š   s    





zSwiGLU.forward)
r,   r-   r.   r/   r   ÚSiLUr   r>   r*   r0   r%   r%   r#   r&   rB   h   s   ørB   c                       s:   e Zd ZdZddejddddf‡ fdd„	Zdd„ Z‡  ZS )	ÚGatedMlpz MLP as used in gMLP
    NTr   c	           
         sÒ   t ƒ  ¡  |p|}|p|}t|ƒ}t|ƒ}	tj|||d d�| _|ƒ | _t |	d ¡| _|d k	r„|d dkspt	‚||ƒ| _
|d }n
t ¡ | _
|d k	rž||ƒnt ¡ | _tj|||d d�| _t |	d ¡| _d S )Nr   r   r2   r   )r   r   r   r   r   r   r   r   r   r4   Úgater   r   r   r   )
r   r   r   r   r   r   Z
gate_layerr   r   r!   r#   r%   r&   r   ˜   s     



zGatedMlp.__init__c                 C   sJ   |   |¡}|  |¡}|  |¡}|  |¡}|  |¡}|  |¡}|  |¡}|S r'   )r   r   r   rG   r   r   r   r(   r%   r%   r&   r*   ¶   s    






zGatedMlp.forwardr+   r%   r%   r#   r&   rF   •   s   ÷rF   c                       s8   e Zd ZdZddejdddf‡ fdd„	Zdd„ Z‡  ZS )	ÚConvMlpz1 MLP using 1x1 convs that keeps spatial dims
    NTr   c                    s€   t ƒ  ¡  |p|}|p|}t|ƒ}tj||d|d d�| _|rF||ƒnt ¡ | _|ƒ | _t 	|¡| _
tj||d|d d�| _d S )Nr   r   )r
   r   )r   r   r   r   r   r   r   r   r   r   r   r   )r   r   r   r   r   r   r   r   r#   r%   r&   r   Ä   s    

zConvMlp.__init__c                 C   s6   |   |¡}|  |¡}|  |¡}|  |¡}|  |¡}|S r'   )r   r   r   r   r   r(   r%   r%   r&   r*   Ù   s    




zConvMlp.forward)	r,   r-   r.   r/   r   ZReLUr   r*   r0   r%   r%   r#   r&   rH   Á   s   ørH   c                       s8   e Zd ZdZddejdddf‡ fdd„	Zdd	„ Z‡  ZS )
ÚGlobalResponseNormMlpzG MLP w/ Global Response Norm (see grn.py), nn.Linear or 1x1 Conv2d
    NTr   Fc           
         s¦   t ƒ  ¡  |p|}|p|}t|ƒ}t|ƒ}|r<ttjdd�ntj}	|	|||d d�| _|ƒ | _t 	|d ¡| _
t|| d�| _|	|||d d�| _t 	|d ¡| _d S )Nr   r	   r   r   )Zchannels_last)r   r   r   r   r   r   r   r   r   r   r   r   Úgrnr   r   )
r   r   r   r   r   r   r   r    r!   r"   r#   r%   r&   r   å   s    

zGlobalResponseNormMlp.__init__c                 C   s@   |   |¡}|  |¡}|  |¡}|  |¡}|  |¡}|  |¡}|S r'   )r   r   r   rJ   r   r   r(   r%   r%   r&   r*   ý   s    





zGlobalResponseNormMlp.forwardr+   r%   r%   r#   r&   rI   â   s   ørI   N)r/   Ú	functoolsr   Ztorchr   rJ   r   Zhelpersr   ÚModuler   r1   rE   ZSwiGLUPackedrB   rF   rH   rI   r%   r%   r%   r&   Ú<module>   s   &2-,!