
    sgt}                     X   d dl Zd dlZd dlmZmZmZmZmZm	Z	 d dl
Z
d dlmZ d dlmZmZmZ ddlmZ ddlmZmZmZ ddlmZ ddlmZmZ dd	lmZmZmZmZm Z  d
dl!m"Z"  ejF                  e$      Z%dZ&dZ' G d dejP                        Z) G d dejP                        Z* G d de      Z+ G d dejP                        Z, G d de,      Z- G d dejP                        Z. G d dejP                        Z/ G d de/      Z0 G d dejP                        Z1 G d  d!ejP                        Z2e/e0d"Z3 G d# d$ejP                        Z4 G d% d&ejP                        Z5 G d' d(ejP                        Z6d)Z7g d*Z8d+Z9 ed,e9       G d- d.e+             Z:d/Z;d0Z< ed1e9       G d2 d3e+             Z=g d4Z>y)5    N)DictListOptionalSetTupleUnion)BCEWithLogitsLossCrossEntropyLossMSELoss   )ACT2FN)BaseModelOutputBaseModelOutputWithPoolingImageClassifierOutput)PreTrainedModel) find_pruneable_heads_and_indicesprune_linear_layer)add_code_sample_docstringsadd_start_docstrings%add_start_docstrings_to_model_forwardlogging	torch_int   )IJepaConfigzfacebook/ijepa_vith14_1kr   c                   `     e Zd ZdZ fdZddej                  dedej                  fdZ xZ	S )IJepaPatchEmbeddingsz
    This class turns `pixel_values` of shape `(batch_size, num_channels, height, width)` into the initial
    `hidden_states` (patch embeddings) of shape `(batch_size, seq_length, hidden_size)` to be consumed by a
    Transformer.
    c                    t         |           |j                  |j                  }}|j                  |j
                  }}t        |t        j                  j                        r|n||f}t        |t        j                  j                        r|n||f}|d   |d   z  |d   |d   z  z  }|| _        || _        || _        || _
        t        j                  ||||      | _        y )Nr   r   )kernel_sizestride)super__init__
image_size
patch_sizenum_channelshidden_size
isinstancecollectionsabcIterablenum_patchesnnConv2d
projection)selfconfigr"   r#   r$   r%   r*   	__class__s          [/var/www/html/venv/lib/python3.12/site-packages/transformers/models/ijepa/modeling_ijepa.pyr!   zIJepaPatchEmbeddings.__init__-   s    !'!2!2F4E4EJ
$*$7$79K9Kk#-j+//:R:R#SZZdfpYq
#-j+//:R:R#SZZdfpYq
!!}
15*Q-:VW=:XY$$(&))L+:^hi    pixel_valuesinterpolate_pos_encodingreturnc                    |j                   \  }}}}|| j                  k7  rt        d| j                   d| d      |sV|| j                  d   k7  s|| j                  d   k7  r2t        d| d| d| j                  d    d| j                  d    d		      | j	                  |      j                  d
      j                  dd
      }|S )NzoMake sure that the channel dimension of the pixel values match with the one set in the configuration. Expected z	 but got .r   r   zInput image size (*z) doesn't match model (z).   )shaper$   
ValueErrorr"   r-   flatten	transpose)r.   r3   r4   
batch_sizer$   heightwidth
embeddingss           r1   forwardzIJepaPatchEmbeddings.forward<   s    2>2D2D/
L&%4,,,!../yaI  (++u8J/J (% 9+,Adooa.@-AE  __\2::1=GG1M
r2   F)
__name__
__module____qualname____doc__r!   torchTensorboolrB   __classcell__r0   s   @r1   r   r   &   s3    jELL D ]b]i]i r2   r   c            	            e Zd ZdZddededdf fdZdej                  de	d	e	dej                  fd
Z
	 	 ddej                  deej                     dedej                  fdZ xZS )IJepaEmbeddingszb
    Construct the CLS token, position and patch embeddings. Optionally, also the mask token.
    r/   use_mask_tokenr5   Nc                    t         |           |r4t        j                  t	        j
                  dd|j                              nd | _        t        |      | _	        | j                  j                  }t        j                  t	        j                  d||j                              | _        t        j                  |j                        | _        |j                   | _        || _        y )Nr   )r    r!   r+   	ParameterrH   zerosr%   
mask_tokenr   patch_embeddingsr*   randnposition_embeddingsDropouthidden_dropout_probdropoutr#   r/   )r.   r/   rO   r*   r0   s       r1   r!   zIJepaEmbeddings.__init__R   s    Q_",,u{{1a9K9K'LMei 4V <++77#%<<A{FL^L^0_#` zz&"<"<= ++r2   rA   r?   r@   c                 0   |j                   d   }| j                  j                   d   }t        j                  j	                         s||k(  r||k(  r| j                  S | j                  }|j                   d   }|| j
                  z  }|| j
                  z  }	t        |dz        }
|j                  d|
|
|      }|j                  dddd      }t        j                  j                  |||	fdd	      }|j                  dddd      j                  dd|      }|S )
a   
        This method allows to interpolate the pre-trained position encodings, to be able to use the model on higher resolution
        images. This method is also adapted to support torch.jit tracing.

        Adapted from:
        - https://github.com/facebookresearch/dino/blob/de9ee3df6cf39fac952ab558447af1fa1365362a/vision_transformer.py#L174-L194, and
        - https://github.com/facebookresearch/dinov2/blob/e1277af2ba9496fbadf7aec6eba56e8d882d1e35/dinov2/models/vision_transformer.py#L179-L211
        r   g      ?r   r   r9   bicubicF)sizemodealign_corners)r:   rV   rH   jit
is_tracingr#   r   reshapepermuter+   
functionalinterpolateview)r.   rA   r?   r@   r*   num_positionspatch_pos_embeddim
new_height	new_widthsqrt_num_positionss              r1   r4   z(IJepaEmbeddings.interpolate_pos_encoding\   s#    !&&q)0066q9 yy##%+*F6UZ?+++22r"t.
T__,	&}c'9:)11!5GI[]`a)11!Q1=--33i(	 4 
 *11!Q1=BB1b#Nr2   r3   bool_masked_posr4   c                 x   |j                   \  }}}}| j                  ||      }|Z|j                   d   }	| j                  j                  ||	d      }
|j	                  d      j                  |
      }|d|z
  z  |
|z  z   }|r|| j                  |||      z   }n|| j                  z   }| j                  |      }|S )N)r4   r   r[         ?)	r:   rT   rS   expand	unsqueezetype_asr4   rV   rY   )r.   r3   rm   r4   r>   _r?   r@   rA   
seq_lengthmask_tokensmasks               r1   rB   zIJepaEmbeddings.forward   s     (4'9'9$
Avu**<Rj*k
&#))!,J//00ZLK",,R088ED#sTz2[45GGJ $#d&C&CJPVX]&^^J#d&>&>>J\\*-
r2   rC   NF)rD   rE   rF   rG   r   rJ   r!   rH   rI   intr4   r   
BoolTensorrB   rK   rL   s   @r1   rN   rN   M   s    { D T %5<< % %UX %]b]i]i %T 7;).	ll "%"2"23 #'	
 
r2   rN   c                       e Zd ZdZeZdZdZdZddgZ	dZ
deej                  ej                  ej                  f   dd	fd
Zy	)IJepaPreTrainedModelz
    An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
    models.
    ijepar3   TrN   
IJepaLayermoduler5   Nc                 
   t        |t        j                  t        j                  f      rt        j                  j                  |j                  j                  j                  t        j                        d| j                  j                        j                  |j                  j                        |j                  _        |j                  %|j                  j                  j                          yyt        |t        j                         rJ|j                  j                  j                          |j                  j                  j#                  d       yt        |t$              rt        j                  j                  |j&                  j                  j                  t        j                        d| j                  j                        j                  |j&                  j                        |j&                  _        yy)zInitialize the weights        )meanstdNro   )r&   r+   Linearr,   inittrunc_normal_weightdatatorH   float32r/   initializer_rangedtypebiaszero_	LayerNormfill_rN   rV   )r.   r~   s     r1   _init_weightsz"IJepaPreTrainedModel._init_weights   sZ   fryy"))45 "$!6!6""%%emm43DKKDaDa "7 "b$$% MM {{&  &&( '-KK""$MM$$S)0.0gg.C.C**//225==AKK11 /D / b++112	 &&+ 1r2   )rD   rE   rF   rG   r   config_classbase_model_prefixmain_input_namesupports_gradient_checkpointing_no_split_modules_supports_sdpar   r+   r   r,   r   r    r2   r1   r{   r{      s[    
 L$O&*#*L9N3E"))RYY*L$M 3RV 3r2   r{   c            
            e Zd Zdeddf fdZdej                  dej                  fdZ	 d
deej                     de	de
eej                  ej                  f   eej                     f   fd	Z xZS )IJepaSelfAttentionr/   r5   Nc                    t         |           |j                  |j                  z  dk7  r3t	        |d      s't        d|j                  f d|j                   d      |j                  | _        t        |j                  |j                  z        | _        | j                  | j                  z  | _        t        j                  |j                  | j                  |j                        | _        t        j                  |j                  | j                  |j                        | _        t        j                  |j                  | j                  |j                        | _        t        j                  |j                         | _        y )Nr   embedding_sizezThe hidden size z4 is not a multiple of the number of attention heads r7   )r   )r    r!   r%   num_attention_headshasattrr;   rx   attention_head_sizeall_head_sizer+   r   qkv_biasquerykeyvaluerW   attention_probs_dropout_probrY   r.   r/   r0   s     r1   r!   zIJepaSelfAttention.__init__   s1    : ::a?PVXhHi"6#5#5#6"7 8334A7 
 $*#=#= #&v'9'9F<V<V'V#W !558P8PPYYv1143E3EFOO\
99V//1C1C&//ZYYv1143E3EFOO\
zz&"E"EFr2   xc                     |j                         d d | j                  | j                  fz   }|j                  |      }|j	                  dddd      S )Nr[   r   r9   r   r   )r]   r   r   rf   rc   )r.   r   new_x_shapes      r1   transpose_for_scoresz'IJepaSelfAttention.transpose_for_scores   sL    ffhsmt'?'?AYAY&ZZFF;yyAq!$$r2   	head_maskoutput_attentionsc                    | j                  |      }| j                  | j                  |            }| j                  | j                  |            }| j                  |      }t	        j
                  ||j                  dd            }|t        j                  | j                        z  }t        j                  j                  |d      }	| j                  |	      }	||	|z  }	t	        j
                  |	|      }
|
j                  dddd      j                         }
|
j!                         d d | j"                  fz   }|
j%                  |      }
|r|
|	f}|S |
f}|S )Nr[   ri   r   r9   r   r   )r   r   r   r   rH   matmulr=   mathsqrtr   r+   rd   softmaxrY   rc   
contiguousr]   r   rf   )r.   hidden_statesr   r   mixed_query_layer	key_layervalue_layerquery_layerattention_scoresattention_probscontext_layernew_context_layer_shapeoutputss                r1   rB   zIJepaSelfAttention.forward   sT    !JJ}5--dhh}.EF	//

=0IJ//0AB !<<Y5H5HR5PQ+dii8P8P.QQ --//0@b/I ,,7  -	9O_kB%--aAq9DDF"/"4"4"6s";t?Q?Q>S"S%**+BC6G=/2 O\M]r2   rw   )rD   rE   rF   r   r!   rH   rI   r   r   rJ   r   r   rB   rK   rL   s   @r1   r   r      s    G{ Gt G$%ell %u|| % bg!(0(>!Z^!	uU\\5<</0%2EE	F!r2   r   c                        e Zd Zdeddf fdZ	 	 d	dej                  deej                     de	de
eej                  ej                  f   eej                     f   f fdZ xZS )
IJepaSdpaSelfAttentionr/   r5   Nc                 F    t         |   |       |j                  | _        y N)r    r!   r   r   s     r1   r!   zIJepaSdpaSelfAttention.__init__   s     ,2,O,O)r2   r   r   r   c           	      ^   |s|'t         j                  d       t        
|   |||      S | j	                  |      }| j                  | j                  |            }| j                  | j                  |            }| j                  |      }t        j                  j                  j                  ||||| j                  r| j                  nddd       }|j                  dddd	      j                         }|j!                         d d
 | j"                  fz   }	|j%                  |	      }|d fS )Na  `IJepaSdpaAttention` is used but `torch.nn.functional.scaled_dot_product_attention` does not support `output_attentions=True` or `head_mask`. Falling back to the manual attention implementation, but specifying the manual implementation will be required from Transformers version v5.0.0 onwards. This warning can be removed using the argument `attn_implementation="eager"` when loading the model.)r   r   r   r   F)	is_causalscaler   r9   r   r   r   )loggerwarning_oncer    rB   r   r   r   r   rH   r+   rd   scaled_dot_product_attentiontrainingr   rc   r   r]   r   rf   )r.   r   r   r   r   r   r   r   r   r   r0   s             r1   rB   zIJepaSdpaSelfAttention.forward  s:    	 5w 7?+#"3 #   !JJ}5--dhh}.EF	//

=0IJ//0AB++HH15D--C I 
 &--aAq9DDF"/"4"4"6s";t?Q?Q>S"S%**+BCd""r2   rw   )rD   rE   rF   r   r!   rH   FloatTensorr   rI   rJ   r   r   rB   rK   rL   s   @r1   r   r      s    P{ Pt P -1"'	'#(('# ELL)'#  	'#
 
uU\\5<</0%2EE	F'# '#r2   r   c                   |     e Zd ZdZdeddf fdZdej                  dej                  dej                  fdZ xZ	S )	IJepaSelfOutputz
    The residual connection is defined in IJepaLayer instead of here (as is the case with other models), due to the
    layernorm applied before each block.
    r/   r5   Nc                     t         |           t        j                  |j                  |j                        | _        t        j                  |j                        | _        y r   )	r    r!   r+   r   r%   denserW   rX   rY   r   s     r1   r!   zIJepaSelfOutput.__init__1  sB    YYv1163E3EF
zz&"<"<=r2   r   input_tensorc                 J    | j                  |      }| j                  |      }|S r   r   rY   r.   r   r   s      r1   rB   zIJepaSelfOutput.forward6  s$    

=1]3r2   )
rD   rE   rF   rG   r   r!   rH   rI   rB   rK   rL   s   @r1   r   r   +  sD    
>{ >t >
U\\  RWR^R^ r2   r   c                        e Zd Zdeddf fdZdee   ddfdZ	 	 ddej                  de
ej                     d	edeeej                  ej                  f   eej                     f   fd
Z xZS )IJepaAttentionr/   r5   Nc                     t         |           t        |      | _        t	        |      | _        t               | _        y r   )r    r!   r   	attentionr   outputsetpruned_headsr   s     r1   r!   zIJepaAttention.__init__>  s0    +F3%f-Er2   headsc                 >   t        |      dk(  ry t        || j                  j                  | j                  j                  | j
                        \  }}t        | j                  j                  |      | j                  _        t        | j                  j                  |      | j                  _        t        | j                  j                  |      | j                  _	        t        | j                  j                  |d      | j                  _        | j                  j                  t        |      z
  | j                  _        | j                  j                  | j                  j                  z  | j                  _        | j
                  j                  |      | _        y )Nr   r   r   )lenr   r   r   r   r   r   r   r   r   r   r   r   union)r.   r   indexs      r1   prune_headszIJepaAttention.prune_headsD  s   u:?74>>55t~~7Y7Y[_[l[l
u
  2$..2F2FN/0B0BEJ1$..2F2FN.t{{/@/@%QO .2^^-O-ORUV[R\-\*'+~~'I'IDNNLnLn'n$ --33E:r2   r   r   r   c                 h    | j                  |||      }| j                  |d   |      }|f|dd  z   }|S )Nr   r   )r   r   )r.   r   r   r   self_outputsattention_outputr   s          r1   rB   zIJepaAttention.forwardV  sE     ~~mY@QR;;|AF#%QR(88r2   rw   )rD   rE   rF   r   r!   r   rx   r   rH   rI   r   rJ   r   r   rB   rK   rL   s   @r1   r   r   =  s    "{ "t ";S ;d ;* -1"'	|| ELL)  	
 
uU\\5<</0%2EE	Fr2   r   c                   (     e Zd Zdeddf fdZ xZS )IJepaSdpaAttentionr/   r5   Nc                 D    t         |   |       t        |      | _        y r   )r    r!   r   r   r   s     r1   r!   zIJepaSdpaAttention.__init__e  s     /7r2   )rD   rE   rF   r   r!   rK   rL   s   @r1   r   r   d  s    8{ 8t 8 8r2   r   c                   `     e Zd Zdeddf fdZdej                  dej                  fdZ xZS )IJepaIntermediater/   r5   Nc                    t         |           t        j                  |j                  |j
                        | _        t        |j                  t              rt        |j                     | _        y |j                  | _        y r   )r    r!   r+   r   r%   intermediate_sizer   r&   
hidden_actstrr   intermediate_act_fnr   s     r1   r!   zIJepaIntermediate.__init__k  s]    YYv1163K3KL
f''-'-f.?.?'@D$'-'8'8D$r2   r   c                 J    | j                  |      }| j                  |      }|S r   )r   r   )r.   r   s     r1   rB   zIJepaIntermediate.forwards  s&    

=100?r2   	rD   rE   rF   r   r!   rH   rI   rB   rK   rL   s   @r1   r   r   j  s1    9{ 9t 9U\\ ell r2   r   c                   x     e Zd Zdeddf fdZdej                  dej                  dej                  fdZ xZS )IJepaOutputr/   r5   Nc                     t         |           t        j                  |j                  |j
                        | _        t        j                  |j                        | _	        y r   )
r    r!   r+   r   r   r%   r   rW   rX   rY   r   s     r1   r!   zIJepaOutput.__init__{  sB    YYv779K9KL
zz&"<"<=r2   r   r   c                 T    | j                  |      }| j                  |      }||z   }|S r   r   r   s      r1   rB   zIJepaOutput.forward  s.    

=1]3%4r2   r   rL   s   @r1   r   r   z  s?    >{ >t >
U\\  RWR^R^ r2   r   )eagersdpac                        e Zd ZdZdeddf fdZ	 	 d
dej                  deej                     de	de
eej                  ej                  f   eej                     f   fd	Z xZS )r}   z?This corresponds to the Block class in the timm implementation.r/   r5   Nc                    t         |           |j                  | _        d| _        t	        |j
                     |      | _        t        |      | _        t        |      | _
        t        j                  |j                  |j                        | _        t        j                  |j                  |j                        | _        y )Nr   eps)r    r!   chunk_size_feed_forwardseq_len_dimIJEPA_ATTENTION_CLASSES_attn_implementationr   r   intermediater   r   r+   r   r%   layer_norm_epslayernorm_beforelayernorm_afterr   s     r1   r!   zIJepaLayer.__init__  s    '-'E'E$01L1LMfU-f5!&) "V-?-?VEZEZ [!||F,>,>FDYDYZr2   r   r   r   c                     | j                  | j                  |      ||      }|d   }|dd  }||z   }| j                  |      }| j                  |      }| j	                  ||      }|f|z   }|S )N)r   r   r   )r   r   r   r   r   )r.   r   r   r   self_attention_outputsr   r   layer_outputs           r1   rB   zIJepaLayer.forward  s     "&!!-0/ "0 "

 2!4(, )=8 ++M:((6 {{<?/G+r2   rw   )rD   rE   rF   rG   r   r!   rH   rI   r   rJ   r   r   rB   rK   rL   s   @r1   r}   r}     s    I[{ [t [ -1"'	|| ELL)  	
 
uU\\5<</0%2EE	Fr2   r}   c                        e Zd Zdeddf fdZ	 	 	 	 ddej                  deej                     deded	ede	e
ef   fd
Z xZS )IJepaEncoderr/   r5   Nc                     t         |           || _        t        j                  t        |j                        D cg c]  }t        |       c}      | _        d| _	        y c c}w rw   )
r    r!   r/   r+   
ModuleListrangenum_hidden_layersr}   layergradient_checkpointing)r.   r/   rs   r0   s      r1   r!   zIJepaEncoder.__init__  sN    ]]fF^F^@_#`1Jv$6#`a
&+# $as   A#r   r   r   output_hidden_statesreturn_dictc                 t   |rdnd }|rdnd }t        | j                        D ]h  \  }}	|r||fz   }|||   nd }
| j                  r+| j                  r| j	                  |	j
                  ||
|      }n
 |	||
|      }|d   }|s`||d   fz   }j |r||fz   }|st        d |||fD              S t        |||      S )Nr   r   r   c              3   &   K   | ]	  }||  y wr   r   ).0vs     r1   	<genexpr>z'IJepaEncoder.forward.<locals>.<genexpr>  s     mq_`_lms   )last_hidden_stater   
attentions)	enumerater	  r
  r   _gradient_checkpointing_func__call__tupler   )r.   r   r   r   r  r  all_hidden_statesall_self_attentionsilayer_modulelayer_head_masklayer_outputss               r1   rB   zIJepaEncoder.forward  s     #7BD$5b4(4 	POA|#$58H$H!.7.CilO**t}} $ A A ))!#%	! !-]OM^ _)!,M &9]1=M<O&O#'	P*   1]4D Dm]4EGZ$[mmm++*
 	
r2   )NFFT)rD   rE   rF   r   r!   rH   rI   r   rJ   r   r  r   rB   rK   rL   s   @r1   r  r    sz    ,{ ,t , -1"'%* )
||)
 ELL))
  	)

 #)
 )
 
uo%	&)
r2   r  c                   *     e Zd Zdef fdZd Z xZS )IJepaPoolerr/   c                     t         |           t        j                  |j                  |j                        | _        t        j                         | _        y r   )r    r!   r+   r   r%   r   Tanh
activationr   s     r1   r!   zIJepaPooler.__init__  s9    YYv1163E3EF
'')r2   c                 \    |d d df   }| j                  |      }| j                  |      }|S )Nr   )r   r"  )r.   r   first_token_tensorpooled_outputs       r1   rB   zIJepaPooler.forward  s6     +1a40

#566r2   )rD   rE   rF   r   r!   rB   rK   rL   s   @r1   r  r    s    ${ $
r2   r  a  
    Args:
        pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):
            Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See [`IJepaImageProcessor.__call__`]
            for details.

        head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*):
            Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`:

            - 1 indicates the head is **not masked**,
            - 0 indicates the head is **masked**.

        output_attentions (`bool`, *optional*):
            Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
            tensors for more detail.
        output_hidden_states (`bool`, *optional*):
            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
            more detail.
        interpolate_pos_encoding (`bool`, *optional*):
            Whether to interpolate the pre-trained position encodings.
        return_dict (`bool`, *optional*):
            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
)r      i   aG  
    This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it
    as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and
    behavior.

    Parameters:
        config ([`IJepaConfig`]): Model configuration class with all the parameters of the model.
            Initializing with a config file does not load the weights associated with the model, only the
            configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
z_The bare IJepa Model transformer outputting raw hidden-states without any specific head on top.c                   8    e Zd Zddededef fdZdefdZdee	e
e	   f   ddfd	Z ee       eeeed
e      	 	 	 	 	 	 	 ddeej*                     deej,                     deej*                     dee   dee   dee   dee   deeef   fd              Z xZS )
IJepaModelr/   add_pooling_layerrO   c                    t         |   |       || _        t        ||      | _        t        |      | _        t        j                  |j                  |j                        | _        |rt        |      nd | _        | j                          y )N)rO   r   )r    r!   r/   rN   rA   r  encoderr+   r   r%   r   	layernormr  pooler	post_init)r.   r/   r)  rO   r0   s       r1   r!   zIJepaModel.__init__&  sk     )&P#F+f&8&8f>S>ST->k&)D 	r2   r5   c                 .    | j                   j                  S r   )rA   rT   )r.   s    r1   get_input_embeddingszIJepaModel.get_input_embeddings2  s    ///r2   heads_to_pruneNc                     |j                         D ]7  \  }}| j                  j                  |   j                  j	                  |       9 y)z
        Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base
        class PreTrainedModel
        N)itemsr+  r	  r   r   )r.   r1  r	  r   s       r1   _prune_headszIJepaModel._prune_heads5  sE    
 +002 	CLE5LLu%//;;EB	Cr2   vision)
checkpointoutput_typer   modalityexpected_outputr3   rm   r   r   r  r4   r  c                    ||n| j                   j                  }||n| j                   j                  }||n| j                   j                  }|t	        d      | j                  || j                   j                        }| j                  j                  j                  j                  j                  }|j                  |k7  r|j                  |      }| j                  |||      }	| j                  |	||||      }
|
d   }| j                  |      }| j                  | j                  |      nd}|s|||fn|f}||
dd z   S t!        |||
j"                  |
j$                        S )z
        bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, num_patches)`, *optional*):
            Boolean masked positions. Indicates which patches are masked (1) and which aren't (0).
        Nz You have to specify pixel_values)rm   r4   )r   r   r  r  r   r   )r  pooler_outputr   r  )r/   r   r  use_return_dictr;   get_head_maskr  rA   rT   r-   r   r   r   r+  r,  r-  r   r   r  )r.   r3   rm   r   r   r  r4   r  expected_dtypeembedding_outputencoder_outputssequence_outputr%  head_outputss                 r1   rB   zIJepaModel.forward=  s   , 2C1N-TXT_T_TqTq$8$D $++JjJj 	 &1%<k$++B]B]?@@ &&y$++2O2OP	 99DDKKQQ/'??>:L??/Tl + 
 ,,/!5# ' 
 *!,..98<8OO4UY?L?XO];_n^pL/!""555)-')77&11	
 	
r2   )FFNNNNNNN)rD   rE   rF   r   rJ   r!   r   r0  r   rx   r   r4  r   IJEPA_INPUTS_DOCSTRINGr   _CHECKPOINT_FOR_DOCr   _CONFIG_FOR_DOC_EXPECTED_OUTPUT_SHAPEr   rH   rI   ry   r   r   rB   rK   rL   s   @r1   r(  r(  !  s"   

{ 
t 
]a 
0&: 0C4T#Y+? CD C ++AB&.$. 046:,0,0/337&*;
u||,;
 "%"2"23;
 ELL)	;

 $D>;
 'tn;
 #+4.;
 d^;
 
u00	1;
 C;
r2   r(  zgoogle/ijepa-base-patch16-224zEgyptian cata  
    IJepa Model transformer with an image classification head on top (a linear layer on top of the final hidden states)
    e.g. for ImageNet.

    <Tip>

        Note that it's possible to fine-tune IJepa on higher resolution images than the ones it has been trained on, by
        setting `interpolate_pos_encoding` to `True` in the forward of the model. This will interpolate the pre-trained
        position embeddings to the higher resolution.

    </Tip>
    c                       e Zd Zdeddf fdZ ee       eee	e
e      	 	 	 	 	 	 	 ddeej                     deej                     deej                     d	ee   d
ee   dee   dee   deee	f   fd              Z xZS )IJepaForImageClassificationr/   r5   Nc                 .   t         |   |       |j                  | _        t        |d      | _        |j                  dkD  r*t        j                  |j                  |j                        nt        j                         | _	        | j                          y )NF)r)  r   )r    r!   
num_labelsr(  r|   r+   r   r%   Identity
classifierr.  r   s     r1   r!   z$IJepaForImageClassification.__init__  ss      ++%@
 OUN_N_bcNc"))F$6$68I8IJikititiv 	r2   )r6  r7  r   r9  r3   r   labelsr   r  r4   r  c                 n   ||n| j                   j                  }| j                  ||||||      }|d   }	| j                  |	j	                  d            }
d}||j                  |
j                        }| j                   j                  | j                  dk(  rd| j                   _        nl| j                  dkD  rL|j                  t        j                  k(  s|j                  t        j                  k(  rd| j                   _        nd| j                   _        | j                   j                  dk(  rIt               }| j                  dk(  r& ||
j                         |j                               }n ||
|      }n| j                   j                  dk(  r=t               } ||
j!                  d	| j                        |j!                  d	            }n,| j                   j                  dk(  rt#               } ||
|      }|s|
f|dd z   }||f|z   S |S t%        ||
|j&                  |j(                  
      S )a  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the image classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        N)r   r   r  r4   r  r   r   r   
regressionsingle_label_classificationmulti_label_classificationr[   )losslogitsr   r  )r/   r<  r|   rM  r   r   deviceproblem_typerK  r   rH   longrx   r   squeezer
   rf   r	   r   r   r  )r.   r3   r   rN  r   r  r4   r  r   rA  rT  rS  loss_fctr   s                 r1   rB   z#IJepaForImageClassification.forward  s   . &1%<k$++B]B]**/!5%=#  
 "!*!5!5!!5!<=YYv}}-F{{''/??a'/;DKK,__q(fllejj.HFLL\a\e\eLe/LDKK,/KDKK,{{''<7"9??a'#FNN$4fnn6FGD#FF3D))-JJ+-B @&++b/R))-II,./Y,F)-)9TGf$EvE$!//))	
 	
r2   rC  )rD   rE   rF   r   r!   r   rD  r   _IMAGE_CLASS_CHECKPOINTr   rF  _IMAGE_CLASS_EXPECTED_OUTPUTr   rH   rI   rJ   r   r  rB   rK   rL   s   @r1   rI  rI    s     
{ 
t 
 ++AB*)$4	 04,0)-,0/337&*A
u||,A
 ELL)A
 &	A

 $D>A
 'tnA
 #+4.A
 d^A
 
u++	,A
 CA
r2   rI  )r{   r(  rI  )?collections.abcr'   r   typingr   r   r   r   r   r   rH   torch.nnr+   r	   r
   r   activationsr   modeling_outputsr   r   r   modeling_utilsr   pytorch_utilsr   r   utilsr   r   r   r   r   configuration_ijepar   
get_loggerrD   r   rE  rF  Moduler   rN   r{   r   r   r   r   r   r   r   r   r}   r  r  rD  rG  IJEPA_START_DOCSTRINGr(  rZ  r[  rI  __all__r   r2   r1   <module>ri     s     : :   A A ! b b - Q  - 
		H	% 1   $299 $NNbii Nb3? 3D9 9x,#/ ,#^bii $$RYY $N8 8		  "))    ' 'T0
299 0
f"))  . ' 	  e[
% [
	[
~ : -   U
"6 U
U
p Pr2   