U
    ¯Ê¹jªK  ã                   @   sœ  d Z ddlZddlZddlmZmZ ddlZddlmZ ddl	m  m
Z ddlmZ ddlmZ ddlmZ ddlmZ eej d	d¡ƒdkZd*eeef eeeef  eejdœdd„Zeeef eedf dœdd„Zd+eedœdd„Zeeef eedf dœdd„ZG dd„ dejƒZ d,eeef eeef dœdd„Z!G d d!„ d!ejƒZ"d-eee d"œd#d$„Z#eeejejejd%œd&d'„Z$G d(d)„ d)ejƒZ%dS ).zf Relative position embedding modules and functions

Hacked together by / Copyright 2022 Ross Wightman
é    N)ÚOptionalÚTupleé   )Úndgrid)ÚRegularGridInterpolator)ÚMlp)Útrunc_normal_ZTIMM_USE_SCIPY_INTERPF)Úq_sizeÚk_sizeÚclass_tokenÚreturnc                 C   sf  |d kst dƒ‚t tt | d ¡t | d ¡ƒ¡ d¡}|d d …d d …d f |d d …d d d …f  }| ddd¡}|d d …d d …df  | d d 7  < |d d …d d …df  | d d 7  < |d d …d d …df  d| d  d 9  < d| d  d d| d  d  }| d¡}|�r^t 	|ddddg¡}||ddd …f< |d |dd …df< |d |d< | 
¡ S )Nz-Different q & k sizes not currently supportedr   r   é   éÿÿÿÿ©r   r   )ÚAssertionErrorÚtorchÚstackr   ÚarangeÚflattenÚpermuteÚsumÚFÚpadÚ
contiguous)r	   r
   r   ZcoordsZrelative_coordsÚnum_relative_distanceÚrelative_position_index© r   úX/var/www/html/bwave/python/venv/lib/python3.8/site-packages/timm/layers/pos_embed_rel.pyÚgen_relative_position_index   s    *,&&* 
r   .)Únew_window_sizeÚnew_bias_shapec                 C   s¢  |d d d |d d d f}| j dkr’|\}}}| j\}}}	||d krX||d ks\t‚||ksl|	|krŽtjjj|  d¡|ddd� d¡} �n| j dks t‚|\}
}| j\}}|
|d |d   }t	|| d ƒ}||f}|d |d k�s|d |d k�rž|�r6| | d …d d …f }| d | …d d …f } nd }tjjj|  
dd¡ dd	|d |d f¡|ddd� d	|
| ¡ 
dd¡} |d k	�ržtj| |fdd
�} | S )Nr   r   r   é   ÚbicubicF)ÚsizeÚmodeZalign_cornersç      à?r   ©Údim)ÚndimÚshaper   r   ÚnnÚ
functionalÚinterpolateÚ	unsqueezeZsqueezeÚintZ	transposeÚreshapeÚviewÚcat)Úrel_pos_biasr   r    Údst_sizeÚ_Údst_hÚdst_wÚnum_attn_headsÚsrc_hÚsrc_wÚdst_num_posÚsrc_num_posÚnum_extra_tokensÚsrc_sizeÚextra_tokensr   r   r   Ú resize_rel_pos_bias_table_simpleI   sP     

üû
$ ü û û
r?   r"   T)ÚinterpolationÚ	antialiasc                 C   s¢   |   ¡ \}}|\}}||ks t‚||krš| j}|  ¡ } t|d ƒ}	t|d ƒ}
tj|  dd¡ d||	|	¡|
|
f||d�}| ||¡ dd¡}| 	|¡ |S | S dS )z’
    Resample relative position bias table suggested in LeVit
    Adapted from: https://github.com/microsoft/Cream/blob/main/TinyViT/utils.py
    r%   r   r   )r#   r$   rA   N)
r#   r   ZdtypeÚfloatr.   r   r,   r   r0   Úto)Zposition_bias_tableÚnew_sizer@   rA   ZL1ZnH1ZL2ZnH2Z
orig_dtypeZS1ZS2Z$relative_position_bias_table_resizedr   r   r   Úresize_rel_pos_bias_table_levitx   s&    
üÿ
rE   c                    sÚ  t rddlm} |d d d |d d d f}| jdkr~d}|\}}}||d kr`||d ksdt‚| j\}	}
}|
|f}d}nP| jdksŒt‚|\}}| j\}}	||d |d   }t|| d ƒ}||f}d}|d |d ksð|d |d k�rÖ|�r$| | d	…d	d	…f }| d	| …d	d	…f } nd	}d
d„ ‰ ‡ fdd„}||d |d ƒ}||d |d ƒ}t |¡t |¡g}|d d }|d d }t 	| |d d¡}t 	| |d d¡}t
||ƒ}g }t|	ƒD ]À}|�rú| d	d	…|f  |d |d ¡ ¡ }n| |d	d	…d	d	…f  ¡ }t �rN|j||| ¡ dd�}t |||ƒ¡ ¡  | j¡}nt||ƒ}||ƒ ¡  | j¡}|�r~| dd¡}| |¡ �qÊ|�r¢tj|dd�} ntj|dd�} |d	k	�rÖ|�sÄt‚tj| |fdd�} | S )a�   Resize relative position bias table using more advanced interpolation.

    Modified from code in Microsoft Unilm (https://github.com/microsoft/unilm) repo (BeiT, BeiT-v2, etc).

    https://github.com/microsoft/unilm/blob/5255d52de86dad642810f5849dd357769346c1d7/beit/run_class_finetuning.py#L351

    Args:
        rel_pos_bias:
        new_window_size:
        new_bias_shape:

    Returns:

    r   )r,   r   r   r!   Fr%   TNc                 S   s   | d||   d|  S )Nç      ð?r   )ÚaÚrÚnr   r   r   Úgeometric_progressionÉ   s    z8resize_rel_pos_bias_table.<locals>.geometric_progressionc           
         sœ   d\}}|| dkrH|| d }ˆ d|| d ƒ}||d krB|}q|}qg }d}t | d ƒD ]}| |¡ |||d  7 }q\dd„ t|ƒD ƒ}	|	dg | S )	N)g)\�Âõ(ð?g      ø?g�íµ ÷Æ°>ç       @r   r   c                 S   s   g | ]
}| ‘qS r   r   )Ú.0r4   r   r   r   Ú
<listcomp>Û   s     z<resize_rel_pos_bias_table.<locals>._calc.<locals>.<listcomp>r   )ÚrangeÚappendÚreversed)
ÚsrcÚdstÚleftÚrightÚqÚgpÚdisÚcurÚiZr_ids©rJ   r   r   Ú_calcÌ   s    
z(resize_rel_pos_bias_table.<locals>._calcrK   gš™™™™™¹?rF   Zcubic)Úkindr   r&   )Ú
_USE_SCIPYZscipyr,   r(   r   r)   r.   r   Ztensorr   r   rN   r0   rB   Zinterp2dÚnumpyÚTensorr   rC   Zdevicer   rO   r1   )r2   r   r    r,   r3   r<   r4   r5   r6   r7   r8   r9   r=   Zhas_flat_shaper:   r;   r>   r[   ÚyÚxZyxÚtyZtxZdyZdxZdyxZall_rel_pos_biasrY   ÚzÚfrH   r   rZ   r   Úresize_rel_pos_bias_table˜   sh     


"
&


re   c                       sP   e Zd ZdZd‡ fdd„	Zdd„ Zejdœdd	„Zde	ej dœdd„Z
‡  ZS )Ú
RelPosBiasz_ Relative Position Bias
    Adapted from Swin-V1 relative position bias impl, modularized.
    r   c                    s²   t ƒ  ¡  |dkst‚|| _|d |d  | _| j| fd |f | _d|d  d d|d  d  d|  }t t 	||¡¡| _
| jdt| j|dkd� d¡dd	� |  ¡  d S )
Nr   r   r   r!   r   )r   r   F©Ú
persistent)ÚsuperÚ__init__r   Úwindow_sizeÚwindow_areaÚ
bias_shaper*   Ú	Parameterr   ÚzerosÚrelative_position_bias_tableÚregister_bufferr   r0   Úinit_weights)Úselfrk   Ú	num_headsÚprefix_tokensr   ©Ú	__class__r   r   rj     s    
(ýzRelPosBias.__init__c                 C   s   t | jdd� d S ©Ng{®Gáz”?)Zstd)r   rp   ©rs   r   r   r   rr   "  s    zRelPosBias.init_weights©r   c                 C   s0   | j | j }| | j¡ ddd¡}| d¡ ¡ S )Nr   r   r   )rp   r   r0   rm   r   r-   r   ©rs   Zrelative_position_biasr   r   r   Úget_bias%  s    zRelPosBias.get_biasN©Úshared_rel_posc                 C   s   ||   ¡  S ©N©r|   ©rs   Zattnr~   r   r   r   Úforward+  s    zRelPosBias.forward)r   )N©Ú__name__Ú
__module__Ú__qualname__Ú__doc__rj   rr   r   r_   r|   r   r‚   Ú__classcell__r   r   rv   r   rf     s
   rf   r   Úswin)Úwin_sizeÚpretrained_win_sizec                 C   sr  |dkst ‚t | d d  | d ¡ tj¡}t | d d  | d ¡ tj¡}t t||ƒ¡}| ddd¡ ¡ }|dk�rR|d dkrÖ|d d …d d …df  |d d   < |d d …d d …df  |d d   < nL|d d …d d …df  | d d   < |d d …d d …df  | d d   < |d9 }t 	|¡t 
d| ¡  ¡ t 
d¡ }nt 	|¡t d| ¡  ¡ }|S )N)r‰   Úcrr   r   r   r‰   é   rF   )r   r   r   rC   Zfloat32r   r   r   r   ÚsignÚlog2ÚabsÚmathÚlog)rŠ   r‹   r$   Zrelative_coords_hZrelative_coords_wZrelative_coords_tabler   r   r   Úgen_relative_log_coords/  s*    ""
&(&&
ÿÿ
ÿr“   c                       sH   e Zd ZdZd‡ fdd„	Zejd	œd
d„Zdeej dœdd„Z	‡  Z
S )Ú	RelPosMlpzß Log-Coordinate Relative Position MLP
    Based on ideas presented in Swin-V2 paper (https://arxiv.org/abs/2111.09883)

    This impl covers the 'swin' implementation as well as two timm specific modes ('cr', and 'rw')
    r�   é€   r   rŒ   r   c                    sÈ   t ƒ  ¡  || _| jd | jd  | _|| _|| _| jfd |f | _|dkrdt ¡ | _	d| _
d}nt ¡ | _	d | _
d}td||tj|dd	�| _| jd
t|ƒ d¡dd� | jdt|||d�dd� d S )Nr   r   r   r‰   é   )TFT)g      À?g        )Zhidden_featuresZout_featuresZ	act_layerZbiasZdropr   r   Frg   Úrel_coords_log)r$   )ri   rj   rk   rl   ru   rt   rm   r*   ZSigmoidÚbias_actÚ	bias_gainZIdentityr   ZReLUÚmlprq   r   r0   r“   )rs   rk   rt   Z
hidden_dimru   r$   Zpretrained_window_sizeZmlp_biasrv   r   r   rj   R  s>    	


ú	ýýzRelPosMlp.__init__rz   c                 C   sŽ   |   | j¡}| jd k	r6| d| j¡| j }| | j¡}| ddd¡}|  |¡}| jd k	rb| j| }| j	r€t
 || j	d| j	dg¡}| d¡ ¡ S )Nr   r   r   r   )rš   r—   r   r0   rt   rm   r   r˜   r™   ru   r   r   r-   r   r{   r   r   r   r|   ~  s    



zRelPosMlp.get_biasNr}   c                 C   s   ||   ¡  S r   r€   r�   r   r   r   r‚   ‹  s    zRelPosMlp.forward)r�   r•   r   rŒ   r   )N)r„   r…   r†   r‡   rj   r   r_   r|   r   r‚   rˆ   r   r   rv   r   r”   L  s        ù,r”   )ÚlengthÚmax_relative_positionc                 C   sv   |dkr| d }d| d }t  | | |¡}t| ƒD ]>}t| ƒD ]0}|| | }t|| ƒ|kr`q>d||||f< q>q2|S )aÏ  Generate a one_hot lookup tensor to reindex embeddings along one dimension.

    Args:
        length: the length to reindex to.
        max_relative_position: the maximum relative position to consider.
            Relative position embeddings for distances above this threshold
            are zeroed out.
    Returns:
        a lookup Tensor of size [length, length, vocab_size] that satisfies
            ret[n,m,v] = 1{m - n + max_relative_position = v}.
    Nr   r   )r   ro   rN   r�   )r›   rœ   Z
vocab_sizeÚretrY   ra   Úvr   r   r   Úgenerate_lookup_tensor�  s    rŸ   )ÚheightÚwidthÚheight_lookupÚwidth_lookupr   c                 C   s8   t  d| |¡}t  d||¡}|| }| | jd ||¡S )a\  Reindex 2d relative position bias with 2 independent einsum lookups.

    Adapted from:
     https://github.com/google-research/maxvit/blob/2e06a7f1f70c76e64cd3dabe5cd1b8c1a23c9fb7/maxvit/models/attention_utils.py

    Args:
        relative_position_tensor: tensor of shape
            [..., vocab_height, vocab_width, ...].
        height: height to reindex to.
        width: width to reindex to.
        height_lookup: one-hot height lookup
        width_lookup: one-hot width lookup
    Returns:
        reindexed_tensor: a Tensor of shape
            [..., height * width, height * width, ...]
    znhw,ixh->nixwznixw,jyw->nijxyr   )r   Zeinsumr/   r)   )Zrelative_position_tensorr    r¡   r¢   r£   Zreindexed_tensorZarear   r   r   Úreindex_2d_einsum_lookup¬  s    r¤   c                       sP   e Zd ZdZd‡ fdd„	Zdd„ Zejdœdd	„Zde	ej dœdd„Z
‡  ZS )ÚRelPosBiasTfzÜ Relative Position Bias Impl (Compatible with Tensorflow MaxViT models)
    Adapted from:
     https://github.com/google-research/maxvit/blob/2e06a7f1f70c76e64cd3dabe5cd1b8c1a23c9fb7/maxvit/models/attention_utils.py
    r   c                    s²   t ƒ  ¡  |dkst‚|| _|d |d  | _|| _d|d  d }d|d  d }| j||f| _t t	 
| j¡¡| _| jdt|d ƒdd� | jdt|d ƒdd� |  ¡  d S )Nr   r   r   r¢   Frg   r£   )ri   rj   r   rk   rl   rt   rm   r*   rn   r   ro   rp   rq   rŸ   rr   )rs   rk   rt   ru   Zvocab_heightZvocab_widthrv   r   r   rj   Î  s    
zRelPosBiasTf.__init__c                 C   s   t jj| jdd� d S rx   )r*   ÚinitZnormal_rp   ry   r   r   r   rr   Ý  s    zRelPosBiasTf.init_weightsrz   c                 C   s"   t | j| jd | jd | j| jƒS )Nr   r   )r¤   rp   rk   r¢   r£   ry   r   r   r   r|   à  s    ûzRelPosBiasTf.get_biasNr}   c                 C   s   ||   ¡  S r   r€   r�   r   r   r   r‚   ê  s    zRelPosBiasTf.forward)r   )Nrƒ   r   r   rv   r   r¥   É  s
   
r¥   )NF)r"   T)r   r‰   )N)&r‡   r‘   ÚosÚtypingr   r   r   Ztorch.nnr*   Ztorch.nn.functionalr+   r   Úgridr   r,   r   rš   r   Zweight_initr   r.   ÚenvironÚgetr]   Úboolr_   r   r?   ÚstrrE   re   ÚModulerf   r“   r”   rŸ   r¤   r¥   r   r   r   r   Ú<module>   sf     ý
ü6

ý2  üü"

ýt%  ý

þE þþú