
    sgn              	          d Z ddlZddlZddlmZ ddlmZmZm	Z	 ddl
Z
ddlZ
ddl
mZ ddlmZ ddlmZmZ dd	lmZ dd
lmZmZmZ ddlmZmZmZmZmZmZ ddlm Z   ejB                  e"      Z#dZ$dZ%g dZ&e G d de             Z'd Z(d Z)dHde
jT                  de+de,de
jT                  fdZ- G d dej\                        Z/ G d dej\                        Z0 G d dej\                        Z1 G d  d!ej\                        Z2 G d" d#ej\                        Z3 G d$ d%ej\                        Z4 G d& d'ej\                        Z5 G d( d)ej\                        Z6 G d* d+ej\                        Z7 G d, d-ej\                        Z8 G d. d/ej\                        Z9 G d0 d1ej\                        Z: G d2 d3ej\                        Z; G d4 d5e      Z<d6Z=d7Z> ed8e=       G d9 d:e<             Z? G d; d<ej\                        Z@ G d= d>ej\                        ZA G d? d@ej\                        ZB G dA dBej\                        ZC G dC dDej\                        ZD edEe=       G dF dGe<             ZEy)Iz"PyTorch Swin2SR Transformer model.    N)	dataclass)OptionalTupleUnion)nn   )ACT2FN)BaseModelOutputImageSuperResolutionOutput)PreTrainedModel) find_pruneable_heads_and_indicesmeshgridprune_linear_layer)ModelOutputadd_code_sample_docstringsadd_start_docstrings%add_start_docstrings_to_model_forwardloggingreplace_return_docstrings   )Swin2SRConfigr   z!caidas/swin2SR-classical-sr-x2-64)r      i  i  c                       e Zd ZU dZdZej                  ed<   dZe	e
ej                        ed<   dZe	e
ej                        ed<   y)Swin2SREncoderOutputa  
    Swin2SR encoder's outputs, with potential hidden states and attentions.

    Args:
        last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
            Sequence of hidden-states at the output of the last layer of the model.
        hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
            Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each stage) of
            shape `(batch_size, sequence_length, hidden_size)`.

            Hidden-states of the model at the output of each layer plus the initial embedding outputs.
        attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
            Tuple of `torch.FloatTensor` (one for each stage) of shape `(batch_size, num_heads, sequence_length,
            sequence_length)`.

            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
            heads.
    Nlast_hidden_statehidden_states
attentions)__name__
__module____qualname____doc__r   torchFloatTensor__annotations__r   r   r   r        _/var/www/html/venv/lib/python3.12/site-packages/transformers/models/swin2sr/modeling_swin2sr.pyr   r   3   sN    & ,0u((/8<M8E%"3"345<59Ju00129r&   r   c                     | j                   \  }}}}| j                  |||z  |||z  ||      } | j                  dddddd      j                         j                  d|||      }|S )z2
    Partitions the given input into windows.
    r   r   r            shapeviewpermute
contiguous)input_featurewindow_size
batch_sizeheightwidthnum_channelswindowss          r'   window_partitionr9   N   s}     /<.A.A+J|!&&Fk);8Lk[gM ##Aq!Q15@@BGGKYdfrsGNr&   c                     | j                   d   }| j                  d||z  ||z  |||      } | j                  dddddd      j                         j                  d|||      } | S )z?
    Merges windows to produce higher resolution features.
    r,   r   r   r   r)   r*   r+   r-   )r8   r3   r5   r6   r7   s        r'   window_reverser;   [   sn     ==$Lll2v4e{6JKYdfrsGooaAq!Q/::<AA"feUabGNr&   input	drop_probtrainingreturnc                    |dk(  s|s| S d|z
  }| j                   d   fd| j                  dz
  z  z   }|t        j                  || j                  | j
                        z   }|j                          | j                  |      |z  }|S )aF  
    Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks).

    Comment by Ross Wightman: This is the same as the DropConnect impl I created for EfficientNet, etc networks,
    however, the original name is misleading as 'Drop Connect' is a different form of dropout in a separate paper...
    See discussion: https://github.com/tensorflow/tpu/issues/494#issuecomment-532968956 ... I've opted for changing the
    layer and argument names to 'drop path' rather than mix DropConnect as a layer name and use 'survival rate' as the
    argument.
            r   r   )r   )dtypedevice)r.   ndimr"   randrB   rC   floor_div)r<   r=   r>   	keep_probr.   random_tensoroutputs          r'   	drop_pathrK   f   s     CxII[[^

Q 77E

5ELL YYMYYy!M1FMr&   c                   x     e Zd ZdZd	dee   ddf fdZdej                  dej                  fdZ	de
fdZ xZS )
Swin2SRDropPathzXDrop paths (Stochastic Depth) per sample (when applied in main path of residual blocks).Nr=   r?   c                 0    t         |           || _        y N)super__init__r=   )selfr=   	__class__s     r'   rQ   zSwin2SRDropPath.__init__~   s    "r&   r   c                 D    t        || j                  | j                        S rO   )rK   r=   r>   rR   r   s     r'   forwardzSwin2SRDropPath.forward   s    FFr&   c                 8    dj                  | j                        S )Nzp={})formatr=   rR   s    r'   
extra_reprzSwin2SRDropPath.extra_repr   s    }}T^^,,r&   rO   )r   r   r    r!   r   floatrQ   r"   TensorrV   strrZ   __classcell__rS   s   @r'   rM   rM   {   sG    b#(5/ #T #GU\\ Gell G-C -r&   rM   c                   f     e Zd ZdZ fdZdeej                     deej                     fdZ
 xZS )Swin2SREmbeddingsz?
    Construct the patch and optional position embeddings.
    c                 x   t         |           t        |      | _        | j                  j                  }|j
                  r=t        j                  t        j                  d|dz   |j                              | _        nd | _        t        j                  |j                        | _        |j                  | _        y )Nr   )rP   rQ   Swin2SRPatchEmbeddingspatch_embeddingsnum_patchesuse_absolute_embeddingsr   	Parameterr"   zeros	embed_dimposition_embeddingsDropouthidden_dropout_probdropoutr3   )rR   configre   rS   s      r'   rQ   zSwin2SREmbeddings.__init__   s     6v >++77))')||EKK;QR?TZTdTd4e'fD$'+D$zz&"<"<=!--r&   pixel_valuesr?   c                     | j                  |      \  }}| j                  || j                  z   }| j                  |      }||fS rO   )rd   rj   rm   )rR   ro   
embeddingsoutput_dimensionss       r'   rV   zSwin2SREmbeddings.forward   sN    (,(=(=l(K%
%##/#d&>&>>J\\*-
,,,r&   )r   r   r    r!   rQ   r   r"   r#   r   r\   rV   r^   r_   s   @r'   ra   ra      s4    .-HU->->$? -E%,,DW -r&   ra   c                   n     e Zd Zd fd	Zdeej                     deej                  ee	   f   fdZ
 xZS )rc   c                    t         |           |j                  }|j                  |j                  }}t        |t        j                  j                        r|n||f}t        |t        j                  j                        r|n||f}|d   |d   z  |d   |d   z  g}|| _	        |d   |d   z  | _
        t        j                  ||j                  ||      | _        |r%t        j                  |j                        | _        y d | _        y )Nr   r   )kernel_sizestride)rP   rQ   ri   
image_size
patch_size
isinstancecollectionsabcIterablepatches_resolutionre   r   Conv2d
projection	LayerNorm	layernorm)rR   rn   normalize_patchesr7   rw   rx   r}   rS   s          r'   rQ   zSwin2SRPatchEmbeddings.__init__   s    ''!'!2!2F4E4EJ
#-j+//:R:R#SZZdfpYq
#-j+//:R:R#SZZdfpYq
(mz!}<jmzZ[}>\]"4-a03Ea3HH))L&2B2BPZcmn;Lf&6&67RVr&   rq   r?   c                     | j                  |      }|j                  \  }}}}||f}|j                  d      j                  dd      }| j                  | j	                  |      }||fS )Nr)   r   )r   r.   flatten	transposer   )rR   rq   _r5   r6   rr   s         r'   rV   zSwin2SRPatchEmbeddings.forward   so    __Z0
(..1fe#UO''*44Q:
>>%
3J,,,r&   )T)r   r   r    rQ   r   r"   r#   r   r\   intrV   r^   r_   s   @r'   rc   rc      s<    W	-(5+<+<"= 	-%V[\_V`H`Ba 	-r&   rc   c                   (     e Zd ZdZ fdZd Z xZS )Swin2SRPatchUnEmbeddingszImage to Patch Unembeddingc                 D    t         |           |j                  | _        y rO   )rP   rQ   ri   )rR   rn   rS   s     r'   rQ   z!Swin2SRPatchUnEmbeddings.__init__   s    ))r&   c                     |j                   \  }}}|j                  dd      j                  || j                  |d   |d         }|S )Nr   r)   r   )r.   r   r/   ri   )rR   rq   x_sizer4   height_widthr7   s         r'   rV   z Swin2SRPatchUnEmbeddings.forward   sO    1;1A1A.
L,))!Q/44ZQWXYQZ\bcd\ef
r&   r   r   r    r!   rQ   rV   r^   r_   s   @r'   r   r      s    %*
r&   r   c            	            e Zd ZdZej
                  fdee   dedej                  ddf fdZ	d Z
d	ej                  d
eeef   dej                  fdZ xZS )Swin2SRPatchMerginga'  
    Patch Merging Layer.

    Args:
        input_resolution (`Tuple[int]`):
            Resolution of input feature.
        dim (`int`):
            Number of input channels.
        norm_layer (`nn.Module`, *optional*, defaults to `nn.LayerNorm`):
            Normalization layer class.
    input_resolutiondim
norm_layerr?   Nc                     t         |           || _        || _        t	        j
                  d|z  d|z  d      | _         |d|z        | _        y )Nr*   r)   Fbias)rP   rQ   r   r   r   Linear	reductionnorm)rR   r   r   r   rS   s       r'   rQ   zSwin2SRPatchMerging.__init__   sI     01s7AG%@q3w'	r&   c                     |dz  dk(  xs |dz  dk(  }|r.ddd|dz  d|dz  f}t         j                  j                  ||      }|S )Nr)   r   r   )r   
functionalpad)rR   r2   r5   r6   
should_pad
pad_valuess         r'   	maybe_padzSwin2SRPatchMerging.maybe_pad   sU    qjAo:519>
Q519a!<JMM--mZHMr&   r2   input_dimensionsc                    |\  }}|j                   \  }}}|j                  ||||      }| j                  |||      }|d d dd ddd dd d f   }|d d dd ddd dd d f   }	|d d dd ddd dd d f   }
|d d dd ddd dd d f   }t        j                  ||	|
|gd      }|j                  |dd|z        }| j                  |      }| j                  |      }|S )Nr   r)   r   r,   r*   )r.   r/   r   r"   catr   r   )rR   r2   r   r5   r6   r4   r   r7   input_feature_0input_feature_1input_feature_2input_feature_3s               r'   rV   zSwin2SRPatchMerging.forward   s   ((5(;(;%
C%**:vulS}feD'14a4Aq(89'14a4Aq(89'14a4Aq(89'14a4Aq(89		?O_Ve"fhjk%**:r1|;KL}5		-0r&   )r   r   r    r!   r   r   r   r   ModulerQ   r   r"   r\   rV   r^   r_   s   @r'   r   r      sr    
 XZWcWc (s (# (299 (hl (U\\ U3PS8_ Y^YeYe r&   r   c                        e Zd Zddgf fd	Zd Z	 	 	 d
dej                  deej                     deej                     dee	   de
ej                     f
d	Z xZS )Swin2SRSelfAttentionr   c           
         t         |           ||z  dk7  rt        d| d| d      || _        t	        ||z        | _        | j                  | j
                  z  | _        t        |t        j                  j                        r|n||f| _        || _        t        j                  t        j                   dt        j"                  |ddf      z              | _        t        j&                  t        j(                  ddd	
      t        j*                  d	      t        j(                  d|d
            | _        t        j.                  | j                  d   dz
   | j                  d   t        j0                        j3                         }t        j.                  | j                  d   dz
   | j                  d   t        j0                        j3                         }t        j4                  t7        ||gd            j9                  ddd      j;                         j=                  d      }|d   dkD  r;|d d d d d d dfxx   |d   dz
  z  cc<   |d d d d d d dfxx   |d   dz
  z  cc<   nS|dkD  rN|d d d d d d dfxx   | j                  d   dz
  z  cc<   |d d d d d d dfxx   | j                  d   dz
  z  cc<   |dz  }t        j>                  |      t        j@                  t        jB                  |      dz         z  tE        j@                  d      z  }|jG                  tI        | j,                  jK                               jL                        }| jO                  d|d       t        j.                  | j                  d         }	t        j.                  | j                  d         }
t        j4                  t7        |	|
gd            }t        jP                  |d      }|d d d d d f   |d d d d d f   z
  }|j9                  ddd      j;                         }|d d d d dfxx   | j                  d   dz
  z  cc<   |d d d d dfxx   | j                  d   dz
  z  cc<   |d d d d dfxx   d| j                  d   z  dz
  z  cc<   |jS                  d      }| jO                  d|d       t        j(                  | j                  | j                  |jT                  
      | _+        t        j(                  | j                  | j                  d
      | _,        t        j(                  | j                  | j                  |jT                  
      | _-        t        j\                  |j^                        | _0        y )Nr   zThe hidden size (z6) is not a multiple of the number of attention heads ()
   r   r)   i   Tr   inplaceFrB   ij)indexing         ?relative_coords_table)
persistentr,   relative_position_index)1rP   rQ   
ValueErrornum_attention_headsr   attention_head_sizeall_head_sizery   rz   r{   r|   r3   pretrained_window_sizer   rg   r"   logoneslogit_scale
Sequentialr   ReLUcontinuous_position_bias_mlparangeint64r[   stackr   r0   r1   	unsqueezesignlog2absmathtonext
parametersrB   register_bufferr   sumqkv_biasquerykeyvaluerk   attention_probs_dropout_probrm   )rR   rn   r   	num_headsr3   r   relative_coords_hrelative_coords_wr   coords_hcoords_wcoordscoords_flattenrelative_coordsr   rS   s                  r'   rQ   zSwin2SRSelfAttention.__init__	  s   ?a#C5(^_h^iijk  $- #&sY#7 !558P8PP%k;??3K3KLKS^`kRl 	 '=#<<		"uzz9aQRBS7T2T(UV,.MMIIa4("''$*?3PY`eAf-
)
 "LL4+;+;A+>+B)CTEUEUVWEX`e`k`klrrt!LL4+;+;A+>+B)CTEUEUVWEX`e`k`klrrtKK"35F!GRVWXWQ1Z\Yq\	 	 "!$q(!!Q1*-1G1JQ1NN-!!Q1*-1G1JQ1NN-1_!!Q1*-1A1A!1Dq1HH-!!Q1*-1A1A!1Dq1HH-"JJ,-

599EZ;[^a;a0bbeienenopeqq 	 !6 8 8d>_>_>j>j>l9m9s9s t46KX]^ << 0 0 34<< 0 0 34Xx&:TJKvq1(At4~aqj7QQ)11!Q:EEG1a D$4$4Q$7!$;; 1a D$4$4Q$7!$;; 1a A(8(8(;$;a$?? "1"5"5b"968O\abYYt1143E3EFOO\
99T//1C1C%PYYt1143E3EFOO\
zz&"E"EFr&   c                     |j                         d d | j                  | j                  fz   }|j                  |      }|j	                  dddd      S )Nr,   r   r)   r   r   )sizer   r   r/   r0   )rR   xnew_x_shapes      r'   transpose_for_scoresz)Swin2SRSelfAttention.transpose_for_scoresF  sL    ffhsmt'?'?AYAY&ZZFF;yyAq!$$r&   r   attention_mask	head_maskoutput_attentionsr?   c                 z   |j                   \  }}}| j                  |      }| j                  | j                  |            }	| j                  | j	                  |            }
| j                  |      }t
        j                  j                  |d      t
        j                  j                  |	d      j                  dd      z  }t        j                  | j                  t        j                  d            j                         }||z  }| j                  | j                         j#                  d| j$                        }|| j&                  j#                  d         j#                  | j(                  d   | j(                  d   z  | j(                  d   | j(                  d   z  d      }|j+                  ddd      j-                         }d	t        j.                  |      z  }||j1                  d      z   }||j                   d   }|j#                  ||z  || j$                  ||      |j1                  d      j1                  d      z   }||j1                  d      j1                  d      z   }|j#                  d| j$                  ||      }t
        j                  j3                  |d      }| j5                  |      }|||z  }t        j6                  ||
      }|j+                  dddd
      j-                         }|j9                         d d | j:                  fz   }|j#                  |      }|r||f}|S |f}|S )Nr,   r   g      Y@)maxr   r   r)      r   )r.   r   r   r   r   r   r   	normalizer   r"   clampr   r   r   expr   r   r/   r   r   r3   r0   r1   sigmoidr   softmaxrm   matmulr   r   )rR   r   r   r   r   r4   r   r7   mixed_query_layer	key_layervalue_layerquery_layerattention_scoresr   relative_position_bias_tablerelative_position_bias
mask_shapeattention_probscontext_layernew_context_layer_shapeoutputss                        r'   rV   zSwin2SRSelfAttention.forwardK  s.    )6(;(;%
C JJ}5--dhh}.EF	//

=0IJ//0AB ==22;B2G"--JaJa2 Kb K

)B
 kk$"2"28LMQQS+k9'+'H'HIcIc'd'i'i(((
$ ">d>Z>Z>_>_`b>c!d!i!iQ$"2"21"55t7G7G7JTM]M]^_M`7`bd"
 "8!?!?1a!H!S!S!U!#emm4J&K!K+.D.N.Nq.QQ%'--a0J/44j(*d6N6NPSUX ((+55a8 9  0.2J2J12M2W2WXY2ZZ/44R9Q9QSVX[\ --//0@b/I ,,7  -	9O_kB%--aAq9DDF"/"4"4"6s";t?Q?Q>S"S%**+BC6G=/2 O\M]r&   NNF)r   r   r    rQ   r   r"   r\   r   r#   boolr   rV   r^   r_   s   @r'   r   r     s    TUWXSY ;Gz% 7;15,1;||; !!2!23; E--.	;
 $D>; 
u||	;r&   r   c                   n     e Zd Z fdZdej
                  dej
                  dej
                  fdZ xZS )Swin2SRSelfOutputc                     t         |           t        j                  ||      | _        t        j
                  |j                        | _        y rO   )rP   rQ   r   r   denserk   r   rm   rR   rn   r   rS   s      r'   rQ   zSwin2SRSelfOutput.__init__  s6    YYsC(
zz&"E"EFr&   r   input_tensorr?   c                 J    | j                  |      }| j                  |      }|S rO   r   rm   )rR   r   r  s      r'   rV   zSwin2SRSelfOutput.forward  s$    

=1]3r&   r   r   r    rQ   r"   r\   rV   r^   r_   s   @r'   r   r     s2    G
U\\  RWR^R^ r&   r   c                        e Zd Zd	 fd	Zd Z	 	 	 d
dej                  deej                     deej                     dee	   de
ej                     f
dZ xZS )Swin2SRAttentionc           
          t         |           t        ||||t        |t        j
                  j                        r|n||f      | _        t        ||      | _	        t               | _        y )Nrn   r   r   r3   r   )rP   rQ   r   ry   rz   r{   r|   rR   r   rJ   setpruned_heads)rR   rn   r   r   r3   r   rS   s         r'   rQ   zSwin2SRAttention.__init__  sc    (#0+//2J2JK $:(*@A
	 (4Er&   c                 >   t        |      dk(  ry t        || j                  j                  | j                  j                  | j
                        \  }}t        | j                  j                  |      | j                  _        t        | j                  j                  |      | j                  _        t        | j                  j                  |      | j                  _	        t        | j                  j                  |d      | j                  _        | j                  j                  t        |      z
  | j                  _        | j                  j                  | j                  j                  z  | j                  _        | j
                  j                  |      | _        y )Nr   r   r   )lenr   rR   r   r   r  r   r   r   r   rJ   r   r   union)rR   headsindexs      r'   prune_headszSwin2SRAttention.prune_heads  s   u:?749900$))2O2OQUQbQb
u
 -TYY__eD		*499==%@		,TYY__eD		.t{{/@/@%QO )-		(E(EE
(R		%"&))"?"?$))B_B_"_		 --33E:r&   r   r   r   r   r?   c                 j    | j                  ||||      }| j                  |d   |      }|f|dd  z   }|S Nr   r   )rR   rJ   )rR   r   r   r   r   self_outputsattention_outputr   s           r'   rV   zSwin2SRAttention.forward  sG     yy	K\];;|AF#%QR(88r&   r   r   )r   r   r    rQ   r  r"   r\   r   r#   r   r   rV   r^   r_   s   @r'   r  r    st    ";* 7;15,1
||
 !!2!23
 E--.	

 $D>
 
u||	
r&   r  c                   V     e Zd Z fdZdej
                  dej
                  fdZ xZS )Swin2SRIntermediatec                    t         |           t        j                  |t	        |j
                  |z              | _        t        |j                  t              rt        |j                     | _        y |j                  | _        y rO   )rP   rQ   r   r   r   	mlp_ratior   ry   
hidden_actr]   r	   intermediate_act_fnr  s      r'   rQ   zSwin2SRIntermediate.__init__  sa    YYsC(8(83(>$?@
f''-'-f.?.?'@D$'-'8'8D$r&   r   r?   c                 J    | j                  |      }| j                  |      }|S rO   )r   r  rU   s     r'   rV   zSwin2SRIntermediate.forward  s&    

=100?r&   r  r_   s   @r'   r  r    s#    9U\\ ell r&   r  c                   V     e Zd Z fdZdej
                  dej
                  fdZ xZS )Swin2SROutputc                     t         |           t        j                  t	        |j
                  |z        |      | _        t        j                  |j                        | _	        y rO   )
rP   rQ   r   r   r   r  r   rk   rl   rm   r  s      r'   rQ   zSwin2SROutput.__init__  sF    YYs6#3#3c#9:C@
zz&"<"<=r&   r   r?   c                 J    | j                  |      }| j                  |      }|S rO   r  rU   s     r'   rV   zSwin2SROutput.forward  s$    

=1]3r&   r  r_   s   @r'   r  r    s#    >
U\\ ell r&   r  c                        e Zd Z	 d fd	Zdeeeef   eeef   f   fdZd Zd Z	 	 dde	j                  deeef   dee	j                     d	ee   dee	j                  e	j                  f   f
d
Z xZS )Swin2SRLayerc           
      n   t         	|           || _        | j                  |j                  |j                  f||f      \  }}|d   | _        |d   | _        t        |||| j                  t        |t        j                  j                        r|n||f      | _        t        j                  ||j                        | _        |dkD  rt!        |      nt        j"                         | _        t'        ||      | _        t+        ||      | _        t        j                  ||j                        | _        y )Nr   r	  epsrA   )rP   rQ   r   _compute_window_shiftr3   
shift_sizer  ry   rz   r{   r|   	attentionr   r   layer_norm_epslayernorm_beforerM   IdentityrK   r  intermediater  rJ   layernorm_after)
rR   rn   r   r   r   drop_path_rater(  r   r3   rS   s
            r'   rQ   zSwin2SRLayer.__init__  s    	 0"&"<"<!3!34z:6N#
Z 'q>$Q-)((0+//2J2JK $:(*@A
 !#Sf6K6K L<JS<P8VXVaVaVc/<#FC0!||CV5J5JKr&   r?   c                     t        | j                  |      D cg c]  \  }}||k  r|n| }}}t        | j                  ||      D cg c]  \  }}}||k  rdn| }}}}||fS c c}}w c c}}}w Nr   )zipr   )rR   target_window_sizetarget_shift_sizerwr3   sr(  s           r'   r'  z"Swin2SRLayer._compute_window_shift  s~    69$:O:OQc6dedaAFq)ee8;D<Q<QS^`q8rssWQ116aq(s
sJ&& fss   A'A-c           	         | j                   dkD  ryt        j                  d||df|      }t        d| j                         t        | j                   | j                          t        | j                    d       f}t        d| j                         t        | j                   | j                          t        | j                    d       f}d}|D ]  }|D ]  }	||d d ||	d d f<   |dz  }  t        || j                        }
|
j                  d| j                  | j                  z        }
|
j                  d      |
j                  d      z
  }|j                  |dk7  t        d            j                  |dk(  t        d            }|S d }|S )Nr   r   r   r,   r)   g      YrA   )
r(  r"   rh   slicer3   r9   r/   r   masked_fillr[   )rR   r5   r6   rB   img_maskheight_sliceswidth_slicescountheight_slicewidth_slicemask_windows	attn_masks               r'   get_attn_maskzSwin2SRLayer.get_attn_mask  s   ??Q{{Avua#8FHa$***+t'''$//)9:t&-M a$***+t'''$//)9:t&-L
 E - #/ K@EHQk1<=QJE
 ,Hd6F6FGL',,R1A1ADDTDT1TUL$..q1L4J4J14MMI!--i1neFmLXXYbfgYginoristI  Ir&   c                     | j                   || j                   z  z
  | j                   z  }| j                   || j                   z  z
  | j                   z  }ddd|d|f}t        j                  j                  ||      }||fS r1  )r3   r   r   r   )rR   r   r5   r6   	pad_right
pad_bottomr   s          r'   r   zSwin2SRLayer.maybe_pad  s    %%0@0@(@@DDTDTT	&&$2B2B)BBdFVFVV
Ay!Z8
))-Dj((r&   r   r   r   r   c                    |\  }}|j                         \  }}}	|}
|j                  ||||	      }| j                  |||      \  }}|j                  \  }}}}| j                  dkD  r1t        j                  || j                   | j                   fd      }n|}t        || j                        }|j                  d| j                  | j                  z  |	      }| j                  |||j                        }||j                  |j                        }| j                  ||||      }|d   }|j                  d| j                  | j                  |	      }t        || j                  ||      }| j                  dkD  r/t        j                  || j                  | j                  fd      }n|}|d   dkD  xs |d   dkD  }|r|d d d |d |d d f   j                         }|j                  |||z  |	      }| j!                  |      }|
| j#                  |      z   }| j%                  |      }| j'                  |      }|| j#                  | j)                  |            z   }|r	||d	   f}|S |f}|S )
Nr   )r   r)   )shiftsdimsr,   r   )r   r   r+   r   )r   r/   r   r.   r(  r"   rollr9   r3   rC  rB   r   rC   r)  r;   r1   r+  rK   r-  rJ   r.  )rR   r   r   r   r   r5   r6   r4   r   channelsshortcutr   
height_pad	width_padshifted_hidden_stateshidden_states_windowsrB  attention_outputsr  attention_windowsshifted_windows
was_paddedlayer_outputlayer_outputss                           r'   rV   zSwin2SRLayer.forward&  s    )"/"4"4"6
Ax  &**:vuhO$(NN=&%$P!z&3&9&9#:y!??Q$)JJ}tFVY]YhYhXhEipv$w!$1! !11FHXHX Y 5 : :2t?O?ORVRbRb?bdl m&&z9MDWDW&X	 !%:%A%ABI NN!9iK\ + 
 -Q/,11"d6F6FHXHXZbc():D<L<LjZcd ??Q %

?DOOUYUdUdCelr s /]Q&;*Q-!*;
 1!WfWfufa2G H S S U-22:v~xX--.?@ 4>>-#@@((7{{<0$t~~d6J6J<6X'YY@Q'8';< YeWfr&   )rA   r   r   NF)r   r   r    rQ   r   r   r'  rC  r   r"   r\   r   r#   r   rV   r^   r_   s   @r'   r#  r#    s    qrL2'eTYZ]_bZbTcejknpsksetTtNu '
8) 26,18||8  S/8 E--.	8
 $D>8 
u||U\\)	*8r&   r#  c                        e Zd ZdZd	 fd	Z	 	 d
dej                  deeef   de	ej                     de	e   deej                     f
dZ xZS )Swin2SRStagezh
    This corresponds to the Residual Swin Transformer Block (RSTB) in the original implementation.
    c                    t         	|           || _        || _        t	        j
                  t        |      D cg c]*  }t        |||||dz  dk(  rdn|j                  dz  |      , c}      | _	        |j                  dk(  rt	        j                  ||ddd      | _        n|j                  dk(  rt	        j                  t	        j                  ||dz  ddd      t	        j                  d	d
      t	        j                  |dz  |dz  ddd      t	        j                  d	d
      t	        j                  |dz  |ddd            | _        t        |d      | _        t#        |      | _        y c c}w )Nr)   r   )rn   r   r   r   r(  r   1convr   r   3convr*   皙?Tnegative_sloper   F)r   )rP   rQ   rn   r   r   
ModuleListranger#  r3   layersresi_connectionr~   convr   	LeakyReLUrc   patch_embedr   patch_unembed)
rR   rn   r   r   depthr   rK   r   irS   s
            r'   rQ   zSwin2SRStage.__init__f  sI   mm u
  !%5'%&UaZqf6H6HA6M+A

 !!W,		#sAq!4DI##w.		#saxAq1C>		#(C1HaA6C>		#(CAq1DI 2&ER5f=7
s   /E.r   r   r   r   r?   c                     |}|\  }}t        | j                        D ]  \  }}	|||   nd }
 |	|||
|      }|d   }  ||||f}| j                  ||      }| j                  |      }| j	                  |      \  }}||z   }||f}|r|dd  z  }|S r  )	enumeraterb  rg  rd  rf  )rR   r   r   r   r   residualr5   r6   ri  layer_modulelayer_head_maskrV  rr   r   stage_outputss                  r'   rV   zSwin2SRStage.forward  s     !((5 	-OA|.7.CilO(8H/[lmM)!,M	- $UFE:**=:JK		-0++M:q%0&(9:]12..Mr&   r  rW  )r   r   r    r!   rQ   r"   r\   r   r   r   r#   r   rV   r^   r_   s   @r'   rY  rY  a  so     >L 26,1||  S/ E--.	
 $D> 
u||	r&   rY  c                        e Zd Z fdZ	 	 	 	 d
dej
                  deeef   deej                     dee
   dee
   dee
   deeef   fd	Z xZS )Swin2SREncoderc                 v   t         |           t        |j                        | _        || _        t        j                  d|j                  t        |j                              D cg c]  }|j                          }}t        j                  t        | j                        D cg c]r  }t        ||j                  |d   |d   f|j                  |   |j                   |   |t        |j                  d |       t        |j                  d |dz           d      t c}      | _        d| _        y c c}w c c}w )Nr   r   )rn   r   r   rh  r   rK   r   F)rP   rQ   r  depths
num_stagesrn   r"   linspacer/  r   itemr   r`  ra  rY  ri   r   stagesgradient_checkpointing)rR   rn   	grid_sizer   dpr	stage_idxrS   s         r'   rQ   zSwin2SREncoder.__init__  s   fmm,!&63H3H#fmmJ\!]^Aqvvx^^mm "'t!7  !((&/lIaL%A --	2$..y9!#fmmJY&?"@3v}}UdW`cdWdGeCfg+,
 ',#! _s   'D1&A7D6r   r   r   r   output_hidden_statesreturn_dictr?   c                    d}|rdnd }|rdnd }	|r||fz  }t        | j                        D ]~  \  }
}|||
   nd }| j                  r,| j                  r | j	                  |j
                  ||||      }n |||||      }|d   }|d   }|d   |d   f}||fz  }|r||fz  }|sw|	|dd  z  }	 |st        d |||	fD              S t        |||	      S )	Nr%   r   r   r   r,   r)   c              3   &   K   | ]	  }||  y wrO   r%   ).0vs     r'   	<genexpr>z)Swin2SREncoder.forward.<locals>.<genexpr>  s     mq_`_lms   r   r   r   )rk  rw  rx  r>   _gradient_checkpointing_func__call__tupler   )rR   r   r   r   r   r|  r}  all_input_dimensionsall_hidden_statesall_self_attentionsri  stage_modulern  rV  rr   s                  r'   rV   zSwin2SREncoder.forward  s6     ""6BD$5b4-!11(5 	9OA|.7.CilO**t}} $ A A ))=:JO]n! !-]<Lo_p q)!,M -a 0 1" 57H7LM %5$77 #!m%55! #}QR'88#)	9, m]4EGZ$[mmm#++*
 	
r&   )NFFT)r   r   r    rQ   r"   r\   r   r   r   r#   r   r   r   rV   r^   r_   s   @r'   rq  rq    s    ,4 26,1/4&*-
||-
  S/-
 E--.	-

 $D>-
 'tn-
 d^-
 
u**	+-
r&   rq  c                   &    e Zd ZdZeZdZdZdZd Z	y)Swin2SRPreTrainedModelz
    An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
    models.
    swin2srro   Tc                 *   t        |t        j                  t        j                  f      rt        j                  j
                  j                  |j                  j                  | j                  j                         |j                  %|j                  j                  j                          yyt        |t        j                        rJ|j                  j                  j                          |j                  j                  j                  d       yy)zInitialize the weights)stdNr   )ry   r   r   r~   r"   inittrunc_normal_weightdatarn   initializer_ranger   zero_r   fill_)rR   modules     r'   _init_weightsz$Swin2SRPreTrainedModel._init_weights  s    fryy"))45HHMM''(:(:@]@]'^{{&  &&( '-KK""$MM$$S) .r&   N)
r   r   r    r!   r   config_classbase_model_prefixmain_input_namesupports_gradient_checkpointingr  r%   r&   r'   r  r    s$    
 !L!$O&*#*r&   r  aJ  
    This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use
    it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and
    behavior.

    Parameters:
        config ([`Swin2SRConfig`]): Model configuration class with all the parameters of the model.
            Initializing with a config file does not load the weights associated with the model, only the
            configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
aN  
    Args:
        pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):
            Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See
            [`Swin2SRImageProcessor.__call__`] for details.
        head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*):
            Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`:

            - 1 indicates the head is **not masked**,
            - 0 indicates the head is **masked**.

        output_attentions (`bool`, *optional*):
            Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
            tensors for more detail.
        output_hidden_states (`bool`, *optional*):
            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
            more detail.
        return_dict (`bool`, *optional*):
            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
zaThe bare Swin2SR Model transformer outputting raw hidden-states without any specific head on top.c                        e Zd Z fdZd Zd Zd Z ee       e	e
eede      	 	 	 	 ddej                  deej                     d	ee   d
ee   dee   deeef   fd              Z xZS )Swin2SRModelc                    t         |   |       || _        |j                  dk(  r>|j                  dk(  r/d}t        j                  |      j                  dddd      | _        nt        j                  dddd      | _        |j                  | _
        t        j                  |j                  |j                  ddd      | _        t        |      | _        t#        || j                   j$                  j&                        | _        t        j*                  |j                  |j,                        | _        t1        |      | _        t        j                  |j                  |j                  ddd      | _        | j7                          y )Nr   )gw#?g8EGr?gB`"?r   )ry  r%  )rP   rQ   rn   r7   num_channels_outr"   r\   r/   meanrh   	img_ranger   r~   ri   first_convolutionra   rq   rq  rd   r}   encoderr   r*  r   r   rg  conv_after_body	post_init)rR   rn   rgb_meanrS   s      r'   rQ   zSwin2SRModel.__init__+  s+    !#(?(?1(D/HX.33Aq!Q?DIAq!Q/DI))!#6+>+>@P@PRSUVXY!Z+F3%f8X8X8k8klf&6&6F<Q<QR5f=!yy)9)96;K;KQPQSTU 	r&   c                 .    | j                   j                  S rO   )rq   rd   rY   s    r'   get_input_embeddingsz!Swin2SRModel.get_input_embeddingsA  s    ///r&   c                     |j                         D ]7  \  }}| j                  j                  |   j                  j	                  |       9 y)z
        Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base
        class PreTrainedModel
        N)itemsr  layerr)  r  )rR   heads_to_pruner  r  s       r'   _prune_headszSwin2SRModel._prune_headsD  sE    
 +002 	CLE5LLu%//;;EB	Cr&   c                 J   |j                         \  }}}}| j                  j                  }|||z  z
  |z  }|||z  z
  |z  }t        j                  j                  |d|d|fd      }| j                  j                  |      | _        || j                  z
  | j                  z  }|S )Nr   reflect)	r   rn   r3   r   r   r   r  type_asr  )rR   ro   r   r5   r6   r3   modulo_pad_heightmodulo_pad_widths           r'   pad_and_normalizezSwin2SRModel.pad_and_normalizeL  s    *//11fe kk--(6K+??;N'%+*==L}}((;KQPa7bdmn II%%l3	$tyy0DNNBr&   vision)
checkpointoutput_typer  modalityexpected_outputro   r   r   r|  r}  r?   c                    ||n| j                   j                  }||n| j                   j                  }||n| j                   j                  }| j	                  |t        | j                   j                              }|j                  \  }}}}| j                  |      }| j                  |      }	| j                  |	      \  }
}| j                  |
|||||      }|d   }| j                  |      }| j                  |||f      }| j                  |      |	z   }|s|f|dd  z   }|S t        ||j                   |j"                        S )Nr   r   r|  r}  r   r   r  )rn   r   r|  use_return_dictget_head_maskr  rs  r.   r  r  rq   r  r   rg  r  r
   r   r   )rR   ro   r   r   r|  r}  r   r5   r6   rq   embedding_outputr   encoder_outputssequence_outputrJ   s                  r'   rV   zSwin2SRModel.forward[  sh     2C1N-TXT_T_TqTq$8$D $++JjJj 	 &1%<k$++B]B] &&y#dkk6H6H2IJ	*001fe --l;++L9
-1__Z-H**,,/!5# ' 
 *!,..9,,_vuoN..?*L%'/!"*==FM-)77&11
 	
r&   )NNNN)r   r   r    rQ   r  r  r  r   SWIN2SR_INPUTS_DOCSTRINGr   _CHECKPOINT_FOR_DOCr
   _CONFIG_FOR_DOC_EXPECTED_OUTPUT_SHAPEr"   r#   r   r   r   r   rV   r^   r_   s   @r'   r  r  &  s    
,0C ++CD&#$. 26,0/3&*5
''5
 E--.5
 $D>	5

 'tn5
 d^5
 
uo%	&5
 E5
r&   r  c                   (     e Zd ZdZ fdZd Z xZS )UpsamplezUpsample module.

    Args:
        scale (`int`):
            Scale factor. Supported scales: 2^n and 3.
        num_features (`int`):
            Channel number of intermediate features.
    c                    t         |           || _        ||dz
  z  dk(  rt        t	        t        j                  |d                  D ]Y  }| j                  d| t        j                  |d|z  ddd             | j                  d| t        j                  d             [ y |dk(  r<t        j                  |d|z  ddd      | _        t        j                  d      | _        y t        d	| d
      )Nr   r   r)   convolution_r*   r   pixelshuffle_	   zScale z/ is not supported. Supported scales: 2^n and 3.)rP   rQ   scalera  r   r   r   
add_moduler   r~   PixelShuffleconvolutionpixelshuffler   )rR   r  num_featuresri  rS   s       r'   rQ   zUpsample.__init__  s    
UQYA%3txxq123 I,qc 2BIIlAP\L\^_abde4fg-s 3R__Q5GHI aZ!yyq<7GAqQD " 2DveW,[\]]r&   c                    | j                   | j                   dz
  z  dk(  rmt        t        t        j                  | j                   d                  D ]6  } | j                  d|       |      } | j                  d|       |      }8 |S | j                   dk(  r"| j                  |      }| j                  |      }|S )Nr   r   r)   r  r  r   )r  ra  r   r   r   __getattr__r  r  )rR   hidden_stateri  s      r'   rV   zUpsample.forward  s    JJ$**q.)a/3txx

A678 SCt//,qc0BCLQDt//-s0CD\RS 	 ZZ1_++L9L,,\:Lr&   r   r_   s   @r'   r  r    s    ^
r&   r  c                   (     e Zd ZdZ fdZd Z xZS )UpsampleOneStepa  UpsampleOneStep module (the difference with Upsample is that it always only has 1conv + 1pixelshuffle)

    Used in lightweight SR to save parameters.

    Args:
        scale (int):
            Scale factor. Supported scales: 2^n and 3.
        in_channels (int):
            Channel number of intermediate features.
        out_channels (int):
            Channel number of output features.
    c                     t         |           t        j                  ||dz  |z  ddd      | _        t        j
                  |      | _        y )Nr)   r   r   )rP   rQ   r   r~   rd  r  pixel_shuffle)rR   r  in_channelsout_channelsrS   s       r'   rQ   zUpsampleOneStep.__init__  s@    IIkE1H+DaAN	__U3r&   c                 J    | j                  |      }| j                  |      }|S rO   )rd  r  )rR   r   s     r'   rV   zUpsampleOneStep.forward  s$    IIaLq!r&   r   r_   s   @r'   r  r    s    4r&   r  c                   $     e Zd Z fdZd Z xZS )PixelShuffleUpsamplerc                 .   t         |           t        j                  |j                  |ddd      | _        t        j                  d      | _        t        |j                  |      | _
        t        j                  ||j                  ddd      | _        y Nr   r   Tr   )rP   rQ   r   r~   ri   conv_before_upsamplere  
activationr  upscaleupsampler  final_convolutionrR   rn   r  rS   s      r'   rQ   zPixelShuffleUpsampler.__init__  ss    $&IIf.>.>aQRTU$V!,,t4 >!#<9P9PRSUVXY!Zr&   c                     | j                  |      }| j                  |      }| j                  |      }| j                  |      }|S rO   )r  r  r  r  )rR   r  r   s      r'   rV   zPixelShuffleUpsampler.forward  sC    %%o6OOAMM!""1%r&   r   r   r    rQ   rV   r^   r_   s   @r'   r  r    s    [r&   r  c                   $     e Zd Z fdZd Z xZS )NearestConvUpsamplerc                    t         |           |j                  dk7  rt        d      t	        j
                  |j                  |ddd      | _        t	        j                  d      | _	        t	        j
                  ||ddd      | _
        t	        j
                  ||ddd      | _        t	        j
                  ||ddd      | _        t	        j
                  ||j                  ddd      | _        t	        j                  dd      | _        y )	Nr*   zNThe nearest+conv upsampler only supports an upscale factor of 4 at the moment.r   r   Tr   r]  r^  )rP   rQ   r  r   r   r~   ri   r  re  r  conv_up1conv_up2conv_hrr  r  lrelur  s      r'   rQ   zNearestConvUpsampler.__init__  s    >>Qmnn$&IIf.>.>aQRTU$V!,,t4		,aAF		,aAFyy|Q1E!#<9P9PRSUVXY!Z\\dC
r&   c           	         | j                  |      }| j                  |      }| j                  | j                  t        j
                  j                  j                  |dd                  }| j                  | j                  t        j
                  j                  j                  |dd                  }| j                  | j                  | j                  |                  }|S )Nr)   nearest)scale_factormode)r  r  r  r  r"   r   r   interpolater  r  r  )rR   r  reconstructions      r'   rV   zNearestConvUpsampler.forward  s    33OD///:**MM%((--99/XY`i9jk
 **MM%((--99/XY`i9jk
 //

4<<;X0YZr&   r  r_   s   @r'   r  r    s    D
r&   r  c                   $     e Zd Z fdZd Z xZS )PixelShuffleAuxUpsamplerc           	      |   t         |           |j                  | _        t        j                  |j
                  |ddd      | _        t        j                  |j                  |ddd      | _        t        j                  d      | _
        t        j                  ||j
                  ddd      | _        t        j                  t        j                  d|ddd      t        j                  d            | _        t        |j                  |      | _        t        j                  ||j                   ddd      | _        y r  )rP   rQ   r  r   r~   r7   conv_bicubicri   r  re  r  conv_auxr   conv_after_auxr  r  r  r  r  s      r'   rQ   z!PixelShuffleAuxUpsampler.__init__	  s    ~~IIf&9&9<AqQ$&IIf.>.>aQRTU$V!,,t4		,0C0CQ1M mmBIIaq!Q,OQSQ]Q]fjQkl >!#<9P9PRSUVXY!Zr&   c                    | j                  |      }| j                  |      }| j                  |      }| j                  |      }| j	                  |      }| j                  |      d d d d d || j                  z  d || j                  z  f   |d d d d d || j                  z  d || j                  z  f   z   }| j                  |      }||fS rO   )r  r  r  r  r  r  r  r  )rR   r  bicubicr5   r6   auxr  s          r'   rV   z PixelShuffleAuxUpsampler.forward  s    ##G,33OD///:mmO,--c2MM/*1a1H6DLL3H1HJ`ETXT`T`L`J`+`aa3ft||335Kut||7K5KKLM 	 //@s""r&   r  r_   s   @r'   r  r    s    
[#r&   r  zm
    Swin2SR Model transformer with an upsampler head on top for image super resolution and restoration.
    c                        e Zd Z fdZ ee       eee      	 	 	 	 	 	 dde	e
j                     de	e
j                     de	e
j                     de	e   de	e   de	e   d	eeef   fd
              Z xZS )Swin2SRForImageSuperResolutionc                 V   t         |   |       t        |      | _        |j                  | _        |j
                  | _        d}| j                  dk(  rt        ||      | _        n| j                  dk(  rt        ||      | _        n| j                  dk(  r1t        |j
                  |j                  |j                        | _        nS| j                  dk(  rt        ||      | _        n2t        j                  |j                  |j                  ddd      | _        | j!                          y )N@   r  pixelshuffle_auxpixelshuffledirectnearest+convr   r   )rP   rQ   r  r  	upsamplerr  r  r  r  r  ri   r  r  r   r~   r  r  r  s      r'   rQ   z'Swin2SRForImageSuperResolution.__init__+  s     #F+))~~ >>^+1&,GDM^^114V\JDM^^33+FNNF<L<LfNeNefDM^^~-0FDM &(YYv/?/?AXAXZ[]^`a%bD" 	r&   )r  r  ro   r   labelsr   r|  r}  r?   c                    ||n| j                   j                  }d}|t        d      |j                  dd \  }}	| j                   j                  dk(  r?t
        j                  j                  ||| j                  z  |	| j                  z  fdd      }
| j                  |||||      }|d	   }| j                  d
v r| j                  |      }nk| j                  dk(  rH| j                  |
||	      \  }}|| j                  j                  z  | j                  j                  z   }n|| j                  |      z   }|| j                  j                  z  | j                  j                  z   }|ddddd|| j                  z  d|	| j                  z  f   }|s|f|dd z   }||f|z   S |S t        |||j                  |j                         S )a  
        Returns:

        Example:
         ```python
         >>> import torch
         >>> import numpy as np
         >>> from PIL import Image
         >>> import requests

         >>> from transformers import AutoImageProcessor, Swin2SRForImageSuperResolution

         >>> processor = AutoImageProcessor.from_pretrained("caidas/swin2SR-classical-sr-x2-64")
         >>> model = Swin2SRForImageSuperResolution.from_pretrained("caidas/swin2SR-classical-sr-x2-64")

         >>> url = "https://huggingface.co/spaces/jjourney1125/swin2sr/resolve/main/samples/butterfly.jpg"
         >>> image = Image.open(requests.get(url, stream=True).raw)
         >>> # prepare image for the model
         >>> inputs = processor(image, return_tensors="pt")

         >>> # forward pass
         >>> with torch.no_grad():
         ...     outputs = model(**inputs)

         >>> output = outputs.reconstruction.data.squeeze().float().cpu().clamp_(0, 1).numpy()
         >>> output = np.moveaxis(output, source=0, destination=-1)
         >>> output = (output * 255.0).round().astype(np.uint8)  # float32 to uint8
         >>> # you can visualize `output` with `Image.fromarray`
         ```Nz'Training is not supported at the momentr)   r  r  F)r   r  align_cornersr  r   )r  r  r  r   )lossr  r   r   )rn   r  NotImplementedErrorr.   r  r   r   r  r  r  r  r  r  r  r   r   r   )rR   ro   r   r  r   r|  r}  r  r5   r6   r  r   r  r  r  rJ   s                   r'   rV   z&Swin2SRForImageSuperResolution.forwardE  s   P &1%<k$++B]B]%&OPP$**12.;;  $66mm//t||+UT\\-AB#	 0 G ,,/!5#  
 "!*>>SS!]]?;N^^11"&--&RW"XNC...1B1BBC)D,B,B?,SSN'$,,*@*@@4<<CTCTT'1.E0E.EG]QUQ]Q]I]G](]^$&4F)-)9TGf$EvE))!//))	
 	
r&   )NNNNNN)r   r   r    rQ   r   r  r   r   r  r   r"   r#   
LongTensorr   r   r   rV   r^   r_   s   @r'   r  r  $  s    4 ++CD+ETcd 5915-1,0/3&*T
u001T
 E--.T
 ))*	T

 $D>T
 'tnT
 d^T
 
u00	1T
 e ET
r&   r  )rA   F)Fr!   collections.abcrz   r   dataclassesr   typingr   r   r   r"   torch.utils.checkpointr   activationsr	   modeling_outputsr
   r   modeling_utilsr   pytorch_utilsr   r   r   utilsr   r   r   r   r   r   configuration_swin2srr   
get_loggerr   loggerr  r  r  r   r9   r;   r\   r[   r   rK   r   rM   ra   rc   r   r   r   r   r  r  r  r#  rY  rq  r  SWIN2SR_START_DOCSTRINGr  r  r  r  r  r  r  r  r%   r&   r'   <module>r     s^   )   ! ) )    ! K - [ [  1 
		H	% " : +  :; : :4	U\\ e T V[VbVb *-bii --		 -<-RYY -6ryy 3")) 3n~299 ~D
		 
+ryy +^"))  	BII 	z299 zzD299 DND
RYY D
N*_ *,	  , gn
) n
	n
b#ryy #Lbii 6BII "299 6#ryy #8  	q
%; q
q
r&   