
    sg              	          d Z ddlZddlZddlmZmZmZmZm	Z	m
Z
 ddlZddlZddlmZ ddlmZmZmZ ddlmZ ddlmZmZmZmZ dd	lmZ dd
lmZmZ ddlmZm Z m!Z!m"Z"m#Z#m$Z$ ddl%m&Z& ddl'm(Z(  e"jR                  e*      Z+dZ,dZ-g dZ.dZ/dZ0 G d dejb                        Z2 G d dejb                        Z3 G d dejb                        Z4 G d de4      Z5 G d dejb                        Z6 G d dejb                        Z7 G d  d!e7      Z8 G d" d#ejb                        Z9dBd$ejt                  d%e;d&e<d'ejt                  fd(Z= G d) d*ejb                        Z> G d+ d,ejb                        Z? G d- d.ejb                        Z@e7e8d/ZA G d0 d1ejb                        ZB G d2 d3ejb                        ZC G d4 d5e      ZDd6ZEd7ZFd8ZG e d9eE       G d: d;eD             ZH e d<eE       G d= d>eD             ZI e d?eE       G d@ dAeDe&             ZJy)CzPyTorch DINOv2 model.    N)DictListOptionalSetTupleUnion)nn)BCEWithLogitsLossCrossEntropyLossMSELoss   )ACT2FN)BackboneOutputBaseModelOutputBaseModelOutputWithPoolingImageClassifierOutput)PreTrainedModel) find_pruneable_heads_and_indicesprune_linear_layer)add_code_sample_docstringsadd_start_docstrings%add_start_docstrings_to_model_forwardloggingreplace_return_docstrings	torch_int)BackboneMixin   )Dinov2Configr   zfacebook/dinov2-base)r   i  i   z(facebook/dinov2-small-imagenet1k-1-layerztabby, tabby catc                        e Zd ZdZdeddf fdZdej                  dededej                  fd	Z	dd
ej                  de
ej                     dej                  fdZ xZS )Dinov2EmbeddingszM
    Construct the CLS token, mask token, position and patch embeddings.
    configreturnNc                 @   t         |           t        j                  t	        j
                  dd|j                              | _        t        j                  t	        j                  d|j                              | _	        t        |      | _        | j                  j                  }t        j                  t	        j
                  d|dz   |j                              | _        t        j                  |j                        | _        |j"                  | _        || _        y )Nr   )super__init__r	   	Parametertorchrandnhidden_size	cls_tokenzeros
mask_tokenDinov2PatchEmbeddingspatch_embeddingsnum_patchesposition_embeddingsDropouthidden_dropout_probdropout
patch_sizer!   )selfr!   r/   	__class__s      ]/var/www/html/venv/lib/python3.12/site-packages/transformers/models/dinov2/modeling_dinov2.pyr%   zDinov2Embeddings.__init__B   s    ekk!Q8J8J&KL,,u{{1f6H6H'IJ 5f =++77#%<<A{QPVPbPb0c#d zz&"<"<= ++    
embeddingsheightwidthc                    |j                   d   dz
  }| j                  j                   d   dz
  }t        j                  j	                         s||k(  r||k(  r| j                  S | j                  ddddf   }| j                  ddddf   }|j                   d   }|| j
                  z  }	|| j
                  z  }
t        |dz        }|j                  d|||      }|j                  dddd      }|j                  }t        j                  j                  |j                  t        j                        |	|
fdd	
      j                  |      }|j                  dddd      j                  dd|      }t        j                   ||fd      S )a-  
        This method allows to interpolate the pre-trained position encodings, to be able to use the model on higher resolution
        images. This method is also adapted to support torch.jit tracing and interpolation at torch.float32 precision.

        Adapted from:
        - https://github.com/facebookresearch/dino/blob/de9ee3df6cf39fac952ab558447af1fa1365362a/vision_transformer.py#L174-L194, and
        - https://github.com/facebookresearch/dinov2/blob/e1277af2ba9496fbadf7aec6eba56e8d882d1e35/dinov2/models/vision_transformer.py#L179-L211
        r   Ng      ?r   r      bicubicF)sizemodealign_cornersdtypedim)shaper0   r'   jit
is_tracingr4   r   reshapepermuterD   r	   
functionalinterpolatetofloat32viewcat)r5   r9   r:   r;   r/   num_positionsclass_pos_embedpatch_pos_embedrF   
new_height	new_widthsqrt_num_positionstarget_dtypes                r7   interpolate_pos_encodingz)Dinov2Embeddings.interpolate_pos_encodingN   s    !&&q)A-0066q9A= yy##%+*F6UZ?+++221bqb59221ab59r"t.
T__,	&}c'9:)11!5GI[]`a)11!Q1=&,,--33u}}-i(	 4 

 "<"
  	 *11!Q1=BB1b#Nyy/?;CCr8   pixel_valuesbool_masked_posc                 ,   |j                   \  }}}}| j                  j                  j                  j                  }| j                  |j                  |            }|Xt        j                  |j                  d      | j                  j                  |j                        j                  d      |      }| j                  j                  |dd      }	t        j                  |	|fd      }|| j                  |||      z   }| j                  |      }|S )NrC   r=   r   r   rE   )rG   r.   
projectionweightrD   rN   r'   where	unsqueezer,   r*   expandrQ   rY   r3   )
r5   rZ   r[   
batch_size_r:   r;   rX   r9   
cls_tokenss
             r7   forwardzDinov2Embeddings.forwardv   s    '3'9'9$
Avu,,77>>DD**<???+NO
&))"-t/A/A*BRBR/S/]/]^_/`blJ
 ^^**:r2>
YY
J7Q?
  $"?"?
FTY"ZZ
\\*-
r8   N)__name__
__module____qualname____doc__r   r%   r'   TensorintrY   r   re   __classcell__r6   s   @r7   r    r    =   s|    
| 
 
&D5<< &D &DUX &D]b]i]i &DPELL 8ELLCY ejeqeq r8   r    c                   Z     e Zd ZdZ fdZdej                  dej                  fdZ xZS )r-   z
    This class turns `pixel_values` of shape `(batch_size, num_channels, height, width)` into the initial
    `hidden_states` (patch embeddings) of shape `(batch_size, seq_length, hidden_size)` to be consumed by a
    Transformer.
    c                    t         |           |j                  |j                  }}|j                  |j
                  }}t        |t        j                  j                        r|n||f}t        |t        j                  j                        r|n||f}|d   |d   z  |d   |d   z  z  }|| _        || _        || _        || _
        t        j                  ||||      | _        y )Nr   r   )kernel_sizestride)r$   r%   
image_sizer4   num_channelsr)   
isinstancecollectionsabcIterabler/   r	   Conv2dr]   )r5   r!   rs   r4   rt   r)   r/   r6   s          r7   r%   zDinov2PatchEmbeddings.__init__   s    !'!2!2F4E4EJ
$*$7$79K9Kk#-j+//:R:R#SZZdfpYq
#-j+//:R:R#SZZdfpYq
!!}
15*Q-:VW=:XY$$(&))L+:^hir8   rZ   r"   c                     |j                   d   }|| j                  k7  rt        d| j                   d| d      | j                  |      j	                  d      j                  dd      }|S )Nr   zoMake sure that the channel dimension of the pixel values match with the one set in the configuration. Expected z	 but got .r>   )rG   rt   
ValueErrorr]   flatten	transpose)r5   rZ   rt   r9   s       r7   re   zDinov2PatchEmbeddings.forward   sz    #))!,4,,,!../yaI  __\2::1=GG1M
r8   )	rg   rh   ri   rj   r%   r'   rk   re   rm   rn   s   @r7   r-   r-      s)    jELL U\\ r8   r-   c            
            e Zd Zdeddf fdZdej                  dej                  fdZ	 d
deej                     de	de
eej                  ej                  f   eej                     f   fd	Z xZS )Dinov2SelfAttentionr!   r"   Nc                    t         |           |j                  |j                  z  dk7  r3t	        |d      s't        d|j                  f d|j                   d      |j                  | _        t        |j                  |j                  z        | _        | j                  | j                  z  | _        t        j                  |j                  | j                  |j                        | _        t        j                  |j                  | j                  |j                        | _        t        j                  |j                  | j                  |j                        | _        t        j                  |j                         | _        y )Nr   embedding_sizezThe hidden size z4 is not a multiple of the number of attention heads r{   bias)r$   r%   r)   num_attention_headshasattrr|   rl   attention_head_sizeall_head_sizer	   Linearqkv_biasquerykeyvaluer1   attention_probs_dropout_probr3   r5   r!   r6   s     r7   r%   zDinov2SelfAttention.__init__   s1    : ::a?PVXhHi"6#5#5#6"7 8334A7 
 $*#=#= #&v'9'9F<V<V'V#W !558P8PPYYv1143E3EFOO\
99V//1C1C&//ZYYv1143E3EFOO\
zz&"E"EFr8   xc                     |j                         d d | j                  | j                  fz   }|j                  |      }|j	                  dddd      S )Nr=   r   r>   r   r   )r@   r   r   rP   rK   )r5   r   new_x_shapes      r7   transpose_for_scoresz(Dinov2SelfAttention.transpose_for_scores   sL    ffhsmt'?'?AYAY&ZZFF;yyAq!$$r8   	head_maskoutput_attentionsc                    | j                  |      }| j                  | j                  |            }| j                  | j                  |            }| j                  |      }t	        j
                  ||j                  dd            }|t        j                  | j                        z  }t        j                  j                  |d      }	| j                  |	      }	||	|z  }	t	        j
                  |	|      }
|
j                  dddd      j                         }
|
j!                         d d | j"                  fz   }|
j%                  |      }
|r|
|	f}|S |
f}|S )Nr=   rE   r   r>   r   r   )r   r   r   r   r'   matmulr~   mathsqrtr   r	   rL   softmaxr3   rK   
contiguousr@   r   rP   )r5   hidden_statesr   r   mixed_query_layer	key_layervalue_layerquery_layerattention_scoresattention_probscontext_layernew_context_layer_shapeoutputss                r7   re   zDinov2SelfAttention.forward   sT    !JJ}5--dhh}.EF	//

=0IJ//0AB !<<Y5H5HR5PQ+dii8P8P.QQ --//0@b/I ,,7  -	9O_kB%--aAq9DDF"/"4"4"6s";t?Q?Q>S"S%**+BC6G=/2 O\M]r8   NF)rg   rh   ri   r   r%   r'   rk   r   r   boolr   r   re   rm   rn   s   @r7   r   r      s    G| G G$%ell %u|| % bg!(0(>!Z^!	uU\\5<</0%2EE	F!r8   r   c            
            e Zd Zdeddf fdZ	 ddeej                     dede	e
ej                  ej                  f   e
ej                     f   f fdZ xZS )	Dinov2SdpaSelfAttentionr!   r"   Nc                 F    t         |   |       |j                  | _        y rf   )r$   r%   r   r   s     r7   r%   z Dinov2SdpaSelfAttention.__init__   s     ,2,O,O)r8   r   r   c           	      Z   |r't         j                  d       t        
|   |||      S | j	                  |      }| j                  | j                  |            }| j                  | j                  |            }| j                  |      }t        j                  j                  j                  ||||| j                  r| j                  nddd       }|j                  dddd	      j                         }|j!                         d d
 | j"                  fz   }	|j%                  |	      }|d fS )Na  Dinov2Model is using Dinov2SdpaSelfAttention, but `torch.nn.functional.scaled_dot_product_attention` does not support `output_attentions=True`. Falling back to the manual attention implementation, but specifying the manual implementation will be required from Transformers version v5.0.0 onwards. This warning can be removed using the argument `attn_implementation="eager"` when loading the model.)r   r   r           F)	is_causalscaler   r>   r   r   r   )loggerwarning_oncer$   re   r   r   r   r   r'   r	   rL   scaled_dot_product_attentiontrainingr   rK   r   r@   r   rP   )r5   r   r   r   r   r   r   r   r   r   r6   s             r7   re   zDinov2SdpaSelfAttention.forward   s4    [ 7?+yTe #   !JJ}5--dhh}.EF	//

=0IJ//0AB++HH15D--C I 
 &--aAq9DDF"/"4"4"6s";t?Q?Q>S"S%**+BCd""r8   r   )rg   rh   ri   r   r%   r   r'   rk   r   r   r   re   rm   rn   s   @r7   r   r      sp    P| P P
 bg!#(0(>!#Z^!#	uU\\5<</0%2EE	F!# !#r8   r   c                   |     e Zd ZdZdeddf fdZdej                  dej                  dej                  fdZ xZ	S )	Dinov2SelfOutputz
    The residual connection is defined in Dinov2Layer instead of here (as is the case with other models), due to the
    layernorm applied before each block.
    r!   r"   Nc                     t         |           t        j                  |j                  |j                        | _        t        j                  |j                        | _        y rf   )	r$   r%   r	   r   r)   denser1   r2   r3   r   s     r7   r%   zDinov2SelfOutput.__init__  sB    YYv1163E3EF
zz&"<"<=r8   r   input_tensorc                 J    | j                  |      }| j                  |      }|S rf   )r   r3   )r5   r   r   s      r7   re   zDinov2SelfOutput.forward  s$    

=1]3r8   )
rg   rh   ri   rj   r   r%   r'   rk   re   rm   rn   s   @r7   r   r     sD    
>| > >
U\\  RWR^R^ r8   r   c                        e Zd Zdeddf fdZdee   ddfdZ	 	 ddej                  de
ej                     d	edeeej                  ej                  f   eej                     f   fd
Z xZS )Dinov2Attentionr!   r"   Nc                     t         |           t        |      | _        t	        |      | _        t               | _        y rf   )r$   r%   r   	attentionr   outputsetpruned_headsr   s     r7   r%   zDinov2Attention.__init__(  s0    ,V4&v.Er8   headsc                 >   t        |      dk(  ry t        || j                  j                  | j                  j                  | j
                        \  }}t        | j                  j                  |      | j                  _        t        | j                  j                  |      | j                  _        t        | j                  j                  |      | j                  _	        t        | j                  j                  |d      | j                  _        | j                  j                  t        |      z
  | j                  _        | j                  j                  | j                  j                  z  | j                  _        | j
                  j                  |      | _        y )Nr   r   rE   )lenr   r   r   r   r   r   r   r   r   r   r   r   union)r5   r   indexs      r7   prune_headszDinov2Attention.prune_heads.  s   u:?74>>55t~~7Y7Y[_[l[l
u
  2$..2F2FN/0B0BEJ1$..2F2FN.t{{/@/@%QO .2^^-O-ORUV[R\-\*'+~~'I'IDNNLnLn'n$ --33E:r8   r   r   r   c                 h    | j                  |||      }| j                  |d   |      }|f|dd  z   }|S )Nr   r   )r   r   )r5   r   r   r   self_outputsattention_outputr   s          r7   re   zDinov2Attention.forward@  sE     ~~mY@QR;;|AF#%QR(88r8   r   )rg   rh   ri   r   r%   r   rl   r   r'   rk   r   r   r   r   re   rm   rn   s   @r7   r   r   '  s    "| " ";S ;d ;* -1"'	|| ELL)  	
 
uU\\5<</0%2EE	Fr8   r   c                   (     e Zd Zdeddf fdZ xZS )Dinov2SdpaAttentionr!   r"   Nc                 D    t         |   |       t        |      | _        y rf   )r$   r%   r   r   r   s     r7   r%   zDinov2SdpaAttention.__init__P  s     08r8   )rg   rh   ri   r   r%   rm   rn   s   @r7   r   r   O  s    9| 9 9 9r8   r   c                   X     e Zd Zd fdZdej
                  dej
                  fdZ xZS )Dinov2LayerScaler"   c                     t         |           t        j                  |j                  t        j                  |j                        z        | _        y rf   )	r$   r%   r	   r&   layerscale_valuer'   onesr)   lambda1r   s     r7   r%   zDinov2LayerScale.__init__V  s8    ||F$;$;ejjI[I[>\$\]r8   hidden_statec                      || j                   z  S rf   )r   r5   r   s     r7   re   zDinov2LayerScale.forwardZ  s    dll**r8   r"   Nrg   rh   ri   r%   r'   rk   re   rm   rn   s   @r7   r   r   U  s$    ^+ELL +U\\ +r8   r   input	drop_probr   r"   c                    |dk(  s|s| S d|z
  }| j                   d   fd| j                  dz
  z  z   }|t        j                  || j                  | j
                        z   }|j                          | j                  |      |z  }|S )aF  
    Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks).

    Comment by Ross Wightman: This is the same as the DropConnect impl I created for EfficientNet, etc networks,
    however, the original name is misleading as 'Drop Connect' is a different form of dropout in a separate paper...
    See discussion: https://github.com/tensorflow/tpu/issues/494#issuecomment-532968956 ... I've opted for changing the
    layer and argument names to 'drop path' rather than mix DropConnect as a layer name and use 'survival rate' as the
    argument.
    r   r   r   )r   )rD   device)rG   ndimr'   randrD   r   floor_div)r   r   r   	keep_probrG   random_tensorr   s          r7   	drop_pathr   _  s     CxII[[^

Q 77E

5ELL YYMYYy!M1FMr8   c                   x     e Zd ZdZd	dee   ddf fdZdej                  dej                  fdZ	de
fdZ xZS )
Dinov2DropPathzXDrop paths (Stochastic Depth) per sample (when applied in main path of residual blocks).Nr   r"   c                 0    t         |           || _        y rf   )r$   r%   r   )r5   r   r6   s     r7   r%   zDinov2DropPath.__init__w  s    "r8   r   c                 D    t        || j                  | j                        S rf   )r   r   r   )r5   r   s     r7   re   zDinov2DropPath.forward{  s    FFr8   c                 8    dj                  | j                        S )Nzp={})formatr   r5   s    r7   
extra_reprzDinov2DropPath.extra_repr~  s    }}T^^,,r8   rf   )rg   rh   ri   rj   r   floatr%   r'   rk   re   strr   rm   rn   s   @r7   r   r   t  sG    b#(5/ #T #GU\\ Gell G-C -r8   r   c                   X     e Zd Zd fdZdej
                  dej
                  fdZ xZS )	Dinov2MLPr"   c                 ~   t         |           |j                  x}}t        |j                  |j                  z        }t        j                  ||d      | _        t        |j                  t              rt        |j                     | _        n|j                  | _        t        j                  ||d      | _        y )NTr   )r$   r%   r)   rl   	mlp_ratior	   r   fc1ru   
hidden_actr   r   
activationfc2r5   r!   in_featuresout_featureshidden_featuresr6   s        r7   r%   zDinov2MLP.__init__  s    %+%7%77lf0063C3CCD99[/Ef''-$V%6%67DO$//DO99_lFr8   r   c                 l    | j                  |      }| j                  |      }| j                  |      }|S rf   )r   r   r   r   s     r7   re   zDinov2MLP.forward  s2    xx-|4xx-r8   r   r   rn   s   @r7   r   r     s$    	GELL U\\ r8   r   c                   X     e Zd Zd fdZdej
                  dej
                  fdZ xZS )Dinov2SwiGLUFFNr"   c                 0   t         |           |j                  x}}t        |j                  |j                  z        }t        |dz  dz        dz   dz  dz  }t        j                  |d|z  d      | _        t        j                  ||d      | _        y )Nr>   r         Tr   )	r$   r%   r)   rl   r   r	   r   
weights_inweights_outr   s        r7   r%   zDinov2SwiGLUFFN.__init__  s    %+%7%77lf0063C3CCD2Q67!;AAE))K_1D4P99_lNr8   r   c                     | j                  |      }|j                  dd      \  }}t        j                  j	                  |      |z  }| j                  |      S )Nr>   r=   rE   )r   chunkr	   rL   silur   )r5   r   x1x2hiddens        r7   re   zDinov2SwiGLUFFN.forward  sS    |4##A2#.B##B'",''r8   r   r   rn   s   @r7   r   r     s$    O(ELL (U\\ (r8   r   )eagersdpac                        e Zd ZdZdeddf fdZ	 	 d
dej                  deej                     de	de
eej                  ej                  f   eej                     f   fd	Z xZS )Dinov2LayerzCThis corresponds to the Block class in the original implementation.r!   r"   Nc                 0   t         |           t        j                  |j                  |j
                        | _        t        |j                     |      | _	        t        |      | _        |j                  dkD  rt        |j                        nt        j                         | _        t        j                  |j                  |j
                        | _        |j"                  rt%        |      | _        nt)        |      | _        t        |      | _        y )Nepsr   )r$   r%   r	   	LayerNormr)   layer_norm_epsnorm1DINOV2_ATTENTION_CLASSES_attn_implementationr   r   layer_scale1drop_path_rater   Identityr   norm2use_swiglu_ffnr   mlpr   layer_scale2r   s     r7   r%   zDinov2Layer.__init__  s    \\&"4"4&:O:OP
1&2M2MNvV,V4BHBWBWZ]B](=(=>cecncncp\\&"4"4&:O:OP
  &v.DH (DH,V4r8   r   r   r   c                 D   | j                  | j                  |      ||      }|d   }| j                  |      }|dd  }| j                  |      |z   }| j	                  |      }| j                  |      }| j                  |      }| j                  |      |z   }|f|z   }|S )N)r   r   r   )r   r  r  r   r  r  r  )r5   r   r   r   self_attention_outputsr   r   layer_outputs           r7   re   zDinov2Layer.forward  s     "&JJ}%/ "0 "

 2!4,,-=>(, '78=H zz-0xx-((6 ~~l3mC/G+r8   r   )rg   rh   ri   rj   r   r%   r'   rk   r   r   r   r   re   rm   rn   s   @r7   r
  r
    s~    M5| 5 5& -1"'	|| ELL)  	
 
uU\\5<</0%2EE	Fr8   r
  c                        e Zd Zdeddf fdZ	 	 	 	 ddej                  deej                     deded	ede	e
ef   fd
Z xZS )Dinov2Encoderr!   r"   Nc                     t         |           || _        t        j                  t        |j                        D cg c]  }t        |       c}      | _        d| _	        y c c}w r   )
r$   r%   r!   r	   
ModuleListrangenum_hidden_layersr
  layergradient_checkpointingr5   r!   rc   r6   s      r7   r%   zDinov2Encoder.__init__  sN    ]]vG_G_A`#aAK$7#ab
&+# $bs   A#r   r   r   output_hidden_statesreturn_dictc                 t   |rdnd }|rdnd }t        | j                        D ]h  \  }}	|r||fz   }|||   nd }
| j                  r+| j                  r| j	                  |	j
                  ||
|      }n
 |	||
|      }|d   }|s`||d   fz   }j |r||fz   }|st        d |||fD              S t        |||      S )N r   r   c              3   &   K   | ]	  }||  y wrf   r)  ).0vs     r7   	<genexpr>z(Dinov2Encoder.forward.<locals>.<genexpr>  s     mq_`_lms   )last_hidden_stater   
attentions)	enumerater#  r$  r   _gradient_checkpointing_func__call__tupler   )r5   r   r   r   r&  r'  all_hidden_statesall_self_attentionsilayer_modulelayer_head_masklayer_outputss               r7   re   zDinov2Encoder.forward  s     #7BD$5b4(4 	POA|#$58H$H!.7.CilO**t}} $ A A ))!#%	! !-]OM^ _)!,M &9]1=M<O&O#'	P*   1]4D Dm]4EGZ$[mmm++*
 	
r8   )NFFT)rg   rh   ri   r   r%   r'   rk   r   r   r   r3  r   re   rm   rn   s   @r7   r  r    sz    ,| , , -1"'%* )
||)
 ELL))
  	)

 #)
 )
 
uo%	&)
r8   r  c                       e Zd ZdZeZdZdZdZdgZ	dZ
deej                  ej                  ej                  f   ddfd	Zy)
Dinov2PreTrainedModelz
    An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
    models.
    dinov2rZ   Tr   moduler"   Nc                 >   t        |t        j                  t        j                  f      rt        j                  j                  |j                  j                  j                  t        j                        d| j                  j                        j                  |j                  j                        |j                  _        |j                  %|j                  j                  j                          yyt        |t        j                         rJ|j                  j                  j                          |j                  j                  j#                  d       yt        |t$              r3t        j                  j                  |j&                  j                  j                  t        j                        d| j                  j                        j                  |j&                  j                        |j&                  _        t        j                  j                  |j(                  j                  j                  t        j                        d| j                  j                        j                  |j(                  j                        |j(                  _        yy)zInitialize the weightsr   )meanstdNg      ?)ru   r	   r   ry   inittrunc_normal_r^   datarN   r'   rO   r!   initializer_rangerD   r   zero_r  fill_r    r0   r*   )r5   r=  s     r7   _init_weightsz#Dinov2PreTrainedModel._init_weights   s   fryy"))45 "$!6!6""%%emm43DKKDaDa "7 "b$$% MM {{&  &&( '-KK""$MM$$S) 01.0gg.C.C**//225==AKK11 /D / b++112	 &&+ %'GG$9$9  %%((7KK11 %: % b!!''(	 ! 2r8   )rg   rh   ri   rj   r   config_classbase_model_prefixmain_input_namesupports_gradient_checkpointing_no_split_modules_supports_sdpar   r	   r   ry   r  rG  r)  r8   r7   r;  r;    sY    
  L $O&*#*+N)E"))RYY*L$M )RV )r8   r;  aH  
    This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it
    as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and
    behavior.

    Parameters:
        config ([`Dinov2Config`]): Model configuration class with all the parameters of the model.
            Initializing with a config file does not load the weights associated with the model, only the
            configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
a4  
    Args:
        pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):
            Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See
            [`BitImageProcessor.preprocess`] for details.

        bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, sequence_length)`):
            Boolean masked positions. Indicates which patches are masked (1) and which aren't (0). Only relevant for
            pre-training.

        head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*):
            Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`:

            - 1 indicates the head is **not masked**,
            - 0 indicates the head is **masked**.

        output_attentions (`bool`, *optional*):
            Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
            tensors for more detail.
        output_hidden_states (`bool`, *optional*):
            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
            more detail.
        return_dict (`bool`, *optional*):
            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
aM  
    Args:
        pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):
            Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See
            [`BitImageProcessor.preprocess`] for details.

        head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*):
            Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`:

            - 1 indicates the head is **not masked**,
            - 0 indicates the head is **masked**.

        output_attentions (`bool`, *optional*):
            Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
            tensors for more detail.
        output_hidden_states (`bool`, *optional*):
            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
            more detail.
        return_dict (`bool`, *optional*):
            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
z`The bare DINOv2 Model transformer outputting raw hidden-states without any specific head on top.c                   "    e Zd Zdef fdZdefdZdeee	e   f   ddfdZ
 ee       eeeede	      	 	 	 	 	 	 dd
eej(                     deej(                     deej(                     dee   dee   dee   deeef   fd              Z xZS )Dinov2Modelr!   c                     t         |   |       || _        t        |      | _        t        |      | _        t        j                  |j                  |j                        | _        | j                          y )Nr  )r$   r%   r!   r    r9   r  encoderr	   r  r)   r  	layernorm	post_initr   s     r7   r%   zDinov2Model.__init__|  sY     *62$V,f&8&8f>S>ST 	r8   r"   c                 .    | j                   j                  S rf   r9   r.   r   s    r7   get_input_embeddingsz Dinov2Model.get_input_embeddings      ///r8   heads_to_pruneNc                     |j                         D ]7  \  }}| j                  j                  |   j                  j	                  |       9 y)z
        Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base
        class PreTrainedModel
        N)itemsrQ  r#  r   r   )r5   rX  r#  r   s       r7   _prune_headszDinov2Model._prune_heads  sE    
 +002 	CLE5LLu%//;;EB	Cr8   vision)
checkpointoutput_typerH  modalityexpected_outputrZ   r[   r   r   r&  r'  c                    ||n| j                   j                  }||n| j                   j                  }||n| j                   j                  }|t	        d      | j                  || j                   j                        }| j                  ||      }| j                  |||||      }|d   }	| j                  |	      }	|	d d dd d f   }
|s|	|
f}||dd  z   S t        |	|
|j                  |j                        S )Nz You have to specify pixel_values)r[   r   r   r&  r'  r   r   )r.  pooler_outputr   r/  )r!   r   r&  use_return_dictr|   get_head_maskr"  r9   rQ  rR  r   r   r/  )r5   rZ   r[   r   r   r&  r'  embedding_outputencoder_outputssequence_outputpooled_outputhead_outputss               r7   re   zDinov2Model.forward  s%   " 2C1N-TXT_T_TqTq$8$D $++JjJj 	 &1%<k$++B]B]?@@ &&y$++2O2OP	??<?Y,,/!5# ' 
 *!,..9'1a0+];L/!""555)-')77&11	
 	
r8   NNNNNN)rg   rh   ri   r   r%   r-   rV  r   rl   r   r[  r   DINOV2_BASE_INPUTS_DOCSTRINGr   _CHECKPOINT_FOR_DOCr   _CONFIG_FOR_DOC_EXPECTED_OUTPUT_SHAPEr   r'   rk   r   r   r   re   rm   rn   s   @r7   rO  rO  w  s    

| 
0&; 0C4T#Y+? CD C ++GH&.$. 0426,0,0/3&*/
u||,/
 "%,,//
 ELL)	/

 $D>/
 'tn/
 d^/
 
u00	1/
 I/
r8   rO  z
    Dinov2 Model transformer with an image classification head on top (a linear layer on top of the final hidden state
    of the [CLS] token) e.g. for ImageNet.
    c                        e Zd Zdeddf fdZ ee       eee	e
e      	 	 	 	 	 	 ddeej                     deej                     deej                     d	ee   d
ee   dee   deee	f   fd              Z xZS )Dinov2ForImageClassificationr!   r"   Nc                 0   t         |   |       |j                  | _        t        |      | _        |j                  dkD  r-t        j                  |j                  dz  |j                        nt        j                         | _	        | j                          y )Nr   r>   )r$   r%   
num_labelsrO  r<  r	   r   r)   r  
classifierrS  r   s     r7   r%   z%Dinov2ForImageClassification.__init__  sy      ++!&) EKDUDUXYDYBIIf((1,f.?.?@_a_j_j_l 	
 	r8   )r]  r^  rH  r`  rZ   r   labelsr   r&  r'  c                    ||n| j                   j                  }| j                  |||||      }|d   }|dddf   }	|ddddf   }
t        j                  |	|
j                  d      gd      }| j                  |      }d}||j                  |j                        }| j                   j                  | j                  dk(  rd| j                   _	        nl| j                  dkD  rL|j                  t        j                  k(  s|j                  t        j                  k(  rd| j                   _	        nd| j                   _	        | j                   j                  dk(  rIt               }| j                  dk(  r& ||j                         |j                               }n |||      }n| j                   j                  dk(  r=t!               } ||j#                  d	| j                        |j#                  d	            }n,| j                   j                  dk(  rt%               } |||      }|s|f|d
d z   }||f|z   S |S t'        |||j(                  |j*                        S )a  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the image classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        Nrb  r   r   rE   
regressionsingle_label_classificationmulti_label_classificationr=   r>   )losslogitsr   r/  )r!   rd  r<  r'   rQ   r?  rt  rN   r   problem_typers  rD   longrl   r   squeezer   rP   r
   r   r   r/  )r5   rZ   r   ru  r   r&  r'  r   rh  r*   patch_tokenslinear_inputr{  rz  loss_fctr   s                   r7   re   z$Dinov2ForImageClassification.forward  s!   , &1%<k$++B]B]++/!5#  
 "!*#AqD)	&q!"u-yy)\->->1->-E!FAN.YYv}}-F{{''/??a'/;DKK,__q(fllejj.HFLL\a\e\eLe/LDKK,/KDKK,{{''<7"9??a'#FNN$4fnn6FGD#FF3D))-JJ+-B @&++b/R))-II,./Y,F)-)9TGf$EvE$!//))	
 	
r8   rk  )rg   rh   ri   r   r%   r   DINOV2_INPUTS_DOCSTRINGr   _IMAGE_CLASS_CHECKPOINTr   rn  _IMAGE_CLASS_EXPECTED_OUTPUTr   r'   rk   r   r   r3  re   rm   rn   s   @r7   rq  rq    s    |   ++BC*)$4	 04,0)-,0/3&*D
u||,D
 ELL)D
 &	D

 $D>D
 'tnD
 d^D
 
u++	,D
 DD
r8   rq  zO
    Dinov2 backbone, to be used with frameworks like DETR and MaskFormer.
    c                        e Zd Z fdZdefdZ ee       ee	e
      	 	 	 d
dej                  dee   dee   dee   de	f
d	              Z xZS )Dinov2Backbonec                 v   t         |   |       t         | 	  |       t        |j                  dz         D cg c]  }|j
                   c}| _        t        |      | _        t        |      | _
        t        j                  |j
                  |j                        | _        | j                          y c c}w )Nr   r  )r$   r%   _init_backboner!  r"  r)   num_featuresr    r9   r  rQ  r	   r  r  rR  rS  r%  s      r7   r%   zDinov2Backbone.__init__8  s     v&9>v?W?WZ[?[9\]AV//]*62$V,f&8&8f>S>ST 	 ^s   B6r"   c                 .    | j                   j                  S rf   rU  r   s    r7   rV  z#Dinov2Backbone.get_input_embeddingsE  rW  r8   )r^  rH  rZ   r&  r   r'  c                 b   ||n| j                   j                  }||n| j                   j                  }||n| j                   j                  }| j	                  |      }| j                  |d||      }|r|j                  n|d   }d}t        | j                  |      D ]  \  }	}
|	| j                  v s| j                   j                  r| j                  |
      }
| j                   j                  rn|
ddddf   }
|j                  \  }}}}| j                   j                  }|
j                  |||z  ||z  d      }
|
j!                  dddd	      j#                         }
||
fz  } |s|r|f|dd z   }|S |f|d	d z   }|S t%        ||r|j                  nd|r|j&                  
      S d
      S )a7  
        Returns:

        Examples:

        ```python
        >>> from transformers import AutoImageProcessor, AutoBackbone
        >>> import torch
        >>> from PIL import Image
        >>> import requests

        >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
        >>> image = Image.open(requests.get(url, stream=True).raw)

        >>> processor = AutoImageProcessor.from_pretrained("facebook/dinov2-base")
        >>> model = AutoBackbone.from_pretrained(
        ...     "facebook/dinov2-base", out_features=["stage2", "stage5", "stage8", "stage11"]
        ... )

        >>> inputs = processor(image, return_tensors="pt")

        >>> outputs = model(**inputs)
        >>> feature_maps = outputs.feature_maps
        >>> list(feature_maps[-1].shape)
        [1, 768, 16, 16]
        ```NT)r&  r   r'  r   r)  r=   r   r   r>   )feature_mapsr   r/  )r!   rd  r&  r   r9   rQ  r   zipstage_namesr   apply_layernormrR  reshape_hidden_statesrG   r4   rJ   rK   r   r   r/  )r5   rZ   r&  r   r'  rf  r   r   r  stager   rb   rc   r:   r;   r4   r   s                    r7   re   zDinov2Backbone.forwardH  s   F &1%<k$++B]B]$8$D $++JjJj 	 2C1N-TXT_T_TqTq??<8,,4K\ju  
 2=--'!*#&t'7'7#G 	0E<)));;..#'>>,#?L;;44#/12#6L 4@3E3E0J65!%!7!7J#/#7#7
FjDXZ_cmZmoq#rL#/#7#71a#C#N#N#PL/	0 #&712;6 M '712;6M%3G'//T->w))
 	
 EI
 	
r8   )NNN)rg   rh   ri   r%   r-   rV  r   r  r   r   rn  r'   rk   r   r   re   rm   rn   s   @r7   r  r  1  s    0&; 0 ++BC>X 04,0&*I
llI
 'tnI
 $D>	I

 d^I
 
I
 Y DI
r8   r  )r   F)Krj   collections.abcrv   r   typingr   r   r   r   r   r   r'   torch.utils.checkpointr	   torch.nnr
   r   r   activationsr   modeling_outputsr   r   r   r   modeling_utilsr   pytorch_utilsr   r   utilsr   r   r   r   r   r   utils.backbone_utilsr   configuration_dinov2r   
get_loggerrg   r   rn  rm  ro  r  r  Moduler    r-   r   r   r   r   r   r   rk   r   r   r   r   r   r   r  r
  r  r;  DINOV2_START_DOCSTRINGrl  r  rO  rq  r  r)  r8   r7   <module>r     sA      : :    A A !  . Q  2 . 
		H	% ! - &  E 1 Lryy L^BII D9")) 9x&#1 &#Tryy &$bii $P9/ 9+ryy +U\\ e T V[VbVb *-RYY -		 &(bii ($  0")) 0h0
BII 0
f%)O %)P	   4 . fO
' O
	O
d  Z
#8 Z
Z
z  	\
*M \
\
r8   