
    sg                     8   d Z ddlZddlmZmZmZmZ ddlZddlZddlm	Z	 ddl
mZmZ ddlmZ ddlmZmZmZ dd	lmZ dd
lmZmZ ddlmZmZmZmZmZ ddlmZ  ej@                  e!      Z"dZ#dZ$ G d de	jJ                        Z& G d de	jJ                        Z' G d de	jJ                        Z( G d de(      Z) G d de	jJ                        Z* G d de	jJ                        Z+ G d de+      Z, G d de	jJ                        Z- G d  d!e	jJ                        Z.e+e,d"Z/ G d# d$e	jJ                        Z0 G d% d&e	jJ                        Z1 G d' d(e	jJ                        Z2 G d) d*e      Z3d+Z4d,Z5 ed-e4       G d. d/e3             Z6 ed0e4       G d1 d2e3             Z7y)3zPyTorch ViViT model.    N)OptionalSetTupleUnion)nn)CrossEntropyLossMSELoss   )ACT2FN)BaseModelOutputBaseModelOutputWithPoolingImageClassifierOutput)PreTrainedModel) find_pruneable_heads_and_indicesprune_linear_layer)add_start_docstrings%add_start_docstrings_to_model_forwardloggingreplace_return_docstrings	torch_int   )VivitConfigzgoogle/vivit-b-16x2-kinetics400r   c                   0     e Zd ZdZ fdZddefdZ xZS )VivitTubeletEmbeddingsa  
    Construct Vivit Tubelet embeddings.

    This module turns a batch of videos of shape (batch_size, num_frames, num_channels, height, width) into a tensor of
    shape (batch_size, seq_len, hidden_size) to be consumed by a Transformer encoder.

    The seq_len (the number of patches) equals (number of frames // tubelet_size[0]) * (height // tubelet_size[1]) *
    (width // tubelet_size[2]).
    c                    t         |           |j                  | _        |j                  | _        |j                  | _        | j                  | j
                  d   z  | j                  | j
                  d   z  z  | j                  | j
                  d   z  z  | _        |j                  | _        t        j                  |j                  |j                  |j                  |j                        | _        y )N   r   r   )kernel_sizestride)super__init__
num_frames
image_sizetubelet_size
patch_sizenum_patcheshidden_size	embed_dimr   Conv3dnum_channels
projectionselfconfig	__class__s     [/var/www/html/venv/lib/python3.12/site-packages/transformers/models/vivit/modeling_vivit.pyr    zVivitTubeletEmbeddings.__init__8   s     ++ ++ --__ 22$//!"446$//!"446 	
  ++))!3!3ATAT]c]p]p
    interpolate_pos_encodingc                 \   |j                   \  }}}}}|sP|| j                  k7  s|| j                  k7  r2t        d| d| d| j                  d    d| j                  d    d	      |j                  ddddd	      }| j	                  |      }|j                  d      j                  dd      }|S )
NzImage image size (*z) doesn't match model (r   r   z).r   r
      )shaper"   
ValueErrorpermuter*   flatten	transpose)	r,   pixel_valuesr1   
batch_sizer!   r)   heightwidthxs	            r/   forwardzVivitTubeletEmbeddings.forwardH   s    >J>P>P;
Jfe'Vt-F%SWSbSbJb$VHAeW 5__Q'($//!*<)=RA  $++Aq!Q:OOL) IIaL""1a(r0   F)__name__
__module____qualname____doc__r    boolr?   __classcell__r.   s   @r/   r   r   -   s    
 d r0   r   c                   p     e Zd ZdZ fdZdej                  dededej                  fdZd
de	fd	Z
 xZS )VivitEmbeddingsz
    Vivit Embeddings.

    Creates embeddings from a video using VivitTubeletEmbeddings, adds CLS token and positional embeddings.
    c                    t         |           t        j                  t	        j
                  dd|j                              | _        t        |      | _	        t        j                  t	        j
                  d| j                  j                  dz   |j                              | _        t        j                  |j                        | _        |j                  dd  | _        || _        y )Nr   )r   r    r   	Parametertorchzerosr&   	cls_tokenr   patch_embeddingsr%   position_embeddingsDropouthidden_dropout_probdropoutr#   r$   r-   r+   s     r/   r    zVivitEmbeddings.__init__a   s    ekk!Q8J8J&KL 6v >#%<<KK400<<q@&BTBTU$
  zz&"<"<= --ab1r0   
embeddingsr<   r=   returnc                    |j                   d   dz
  }| j                  j                   d   dz
  }t        j                  j	                         s||k(  r||k(  r| j                  S | j                  ddddf   }| j                  ddddf   }|j                   d   }|| j
                  d   z  }	|| j
                  d   z  }
t        |dz        }|j                  d|||      }|j                  dddd      }t        j                  j                  ||	|
fdd	
      }|j                  dddd      j                  dd|      }t        j                  ||fd      S )a   
        This method allows to interpolate the pre-trained position encodings, to be able to use the model on higher resolution
        images. This method is also adapted to support torch.jit tracing.

        Adapted from:
        - https://github.com/facebookresearch/dino/blob/de9ee3df6cf39fac952ab558447af1fa1365362a/vision_transformer.py#L174-L194, and
        - https://github.com/facebookresearch/dinov2/blob/e1277af2ba9496fbadf7aec6eba56e8d882d1e35/dinov2/models/vision_transformer.py#L179-L211
        r   Nr   g      ?r
   r   bicubicF)sizemodealign_cornersdim)r5   rP   rL   jit
is_tracingr$   r   reshaper7   r   
functionalinterpolateviewcat)r,   rT   r<   r=   r%   num_positionsclass_pos_embedpatch_pos_embedr]   
new_height	new_widthsqrt_num_positionss               r/   r1   z(VivitEmbeddings.interpolate_pos_encodingo   sj    !&&q)A-0066q9A= yy##%+*F6UZ?+++221bqb59221ab59r"tq11
T__Q//	&}c'9:)11!5GI[]`a)11!Q1=--33i(	 4 
 *11!Q1=BB1b#Nyy/?;CCr0   r1   c                 0   |j                   \  }}}}}| j                  ||      }| j                  j                  |ddg      }	t	        j
                  |	|fd      }|r|| j                  |||      z   }n|| j                  z   }| j                  |      }|S )Nr1   r   r\   )	r5   rO   rN   tilerL   rd   r1   rP   rS   )
r,   r:   r1   r;   r!   r)   r<   r=   rT   
cls_tokenss
             r/   r?   zVivitEmbeddings.forward   s    >J>P>P;
Jfe**<Rj*k
^^((*a);<
YY
J7Q?
 $#d&C&CJPVX]&^^J#d&>&>>J\\*-
r0   r@   )rA   rB   rC   rD   r    rL   Tensorintr1   rE   r?   rF   rG   s   @r/   rI   rI   Z   sL    &D5<< &D &DUX &D]b]i]i &DPd r0   rI   c            
            e Zd Zdeddf fdZdej                  dej                  fdZ	 d
deej                     de	de
eej                  ej                  f   eej                     f   fd	Z xZS )VivitSelfAttentionr-   rU   Nc                    t         |           |j                  |j                  z  dk7  r3t	        |d      s't        d|j                  f d|j                   d      |j                  | _        t        |j                  |j                  z        | _        | j                  | j                  z  | _        t        j                  |j                  | j                  |j                        | _        t        j                  |j                  | j                  |j                        | _        t        j                  |j                  | j                  |j                        | _        t        j                  |j                         | _        y )Nr   embedding_sizezThe hidden size z4 is not a multiple of the number of attention heads .)bias)r   r    r&   num_attention_headshasattrr6   rp   attention_head_sizeall_head_sizer   Linearqkv_biasquerykeyvaluerQ   attention_probs_dropout_probrS   r+   s     r/   r    zVivitSelfAttention.__init__   s1    : ::a?PVXhHi"6#5#5#6"7 8334A7 
 $*#=#= #&v'9'9F<V<V'V#W !558P8PPYYv1143E3EFOO\
99V//1C1C&//ZYYv1143E3EFOO\
zz&"E"EFr0   r>   c                     |j                         d d | j                  | j                  fz   }|j                  |      }|j	                  dddd      S )NrW   r   r   r   r
   )rY   rw   ry   rc   r7   )r,   r>   new_x_shapes      r/   transpose_for_scoresz'VivitSelfAttention.transpose_for_scores   sL    ffhsmt'?'?AYAY&ZZFF;yyAq!$$r0   	head_maskoutput_attentionsc                    | j                  |      }| j                  | j                  |            }| j                  | j                  |            }| j                  |      }t	        j
                  ||j                  dd            }|t        j                  | j                        z  }t        j                  j                  |d      }	| j                  |	      }	||	|z  }	t	        j
                  |	|      }
|
j                  dddd      j                         }
|
j!                         d d | j"                  fz   }|
j%                  |      }
|r|
|	f}|S |
f}|S )NrW   r\   r   r   r   r
   )r}   r   r~   r   rL   matmulr9   mathsqrtry   r   ra   softmaxrS   r7   
contiguousrY   rz   rc   )r,   hidden_statesr   r   mixed_query_layer	key_layervalue_layerquery_layerattention_scoresattention_probscontext_layernew_context_layer_shapeoutputss                r/   r?   zVivitSelfAttention.forward   sT    !JJ}5--dhh}.EF	//

=0IJ//0AB !<<Y5H5HR5PQ+dii8P8P.QQ --//0@b/I ,,7  -	9O_kB%--aAq9DDF"/"4"4"6s";t?Q?Q>S"S%**+BC6G=/2 O\M]r0   NF)rA   rB   rC   r   r    rL   ro   r   r   rE   r   r   r?   rF   rG   s   @r/   rr   rr      s    G{ Gt G$%ell %u|| % bg!(0(>!Z^!	uU\\5<</0%2EE	F!r0   rr   c            
            e Zd Zdeddf fdZ	 ddeej                     dede	e
ej                  ej                  f   e
ej                     f   f fdZ xZS )	VivitSdpaSelfAttentionr-   rU   Nc                 F    t         |   |       |j                  | _        y N)r   r    r   r+   s     r/   r    zVivitSdpaSelfAttention.__init__   s     ,2,O,O)r0   r   r   c           	      \   |s|&t         j                  d       t        
|   |||      S | j	                  |      }| j                  | j                  |            }| j                  | j                  |            }| j                  |      }t        j                  j                  j                  ||||| j                  r| j                  nddd       }|j                  dddd      j                         }|j!                         d d	 | j"                  fz   }	|j%                  |	      }|d fS )
Na  VivitSdpaSelfAttention is used but `torch.nn.functional.scaled_dot_product_attention` does not support `output_attentions=True` or `head_mask`. Falling back to the manual attention implementation, but specifying the manual implementation will be required from Transformers version v5.0.0 onwards. This warning can be removed using the argument `attn_implementation="eager"` when loading the model.        F)	is_causalscaler   r   r   r
   r   )loggerwarning_oncer   r?   r}   r   r~   r   rL   r   ra   scaled_dot_product_attentiontrainingr   r7   r   rY   rz   rc   )r,   r   r   r   r   r   r   r   r   r   r.   s             r/   r?   zVivitSdpaSelfAttention.forward   s5    	 5d 7?!  !JJ}5--dhh}.EF	//

=0IJ//0AB++HH15D--C I 
 &--aAq9DDF"/"4"4"6s";t?Q?Q>S"S%**+BCd""r0   r   )rA   rB   rC   r   r    r   rL   ro   rE   r   r   r?   rF   rG   s   @r/   r   r      sp    P{ Pt P
 bg$#(0(>$#Z^$#	uU\\5<</0%2EE	F$# $#r0   r   c                   |     e Zd ZdZdeddf fdZdej                  dej                  dej                  fdZ xZ	S )	VivitSelfOutputz
    The residual connection is defined in VivitLayer instead of here (as is the case with other models), due to the
    layernorm applied before each block.
    r-   rU   Nc                     t         |           t        j                  |j                  |j                        | _        t        j                  |j                        | _        y r   )	r   r    r   r{   r&   denserQ   rR   rS   r+   s     r/   r    zVivitSelfOutput.__init__  sB    YYv1163E3EF
zz&"<"<=r0   r   input_tensorc                 J    | j                  |      }| j                  |      }|S r   r   rS   r,   r   r   s      r/   r?   zVivitSelfOutput.forward  s$    

=1]3r0   )
rA   rB   rC   rD   r   r    rL   ro   r?   rF   rG   s   @r/   r   r     sD    
>{ >t >
U\\  RWR^R^ r0   r   c                        e Zd Zdeddf fdZdee   ddfdZ	 	 ddej                  de
ej                     d	edeeej                  ej                  f   eej                     f   fd
Z xZS )VivitAttentionr-   rU   Nc                     t         |           t        |      | _        t	        |      | _        t               | _        y r   )r   r    rr   	attentionr   outputsetpruned_headsr+   s     r/   r    zVivitAttention.__init__(  s0    +F3%f-Er0   headsc                 >   t        |      dk(  ry t        || j                  j                  | j                  j                  | j
                        \  }}t        | j                  j                  |      | j                  _        t        | j                  j                  |      | j                  _        t        | j                  j                  |      | j                  _	        t        | j                  j                  |d      | j                  _        | j                  j                  t        |      z
  | j                  _        | j                  j                  | j                  j                  z  | j                  _        | j
                  j                  |      | _        y )Nr   r   r\   )lenr   r   rw   ry   r   r   r}   r~   r   r   r   rz   union)r,   r   indexs      r/   prune_headszVivitAttention.prune_heads.  s   u:?74>>55t~~7Y7Y[_[l[l
u
  2$..2F2FN/0B0BEJ1$..2F2FN.t{{/@/@%QO .2^^-O-ORUV[R\-\*'+~~'I'IDNNLnLn'n$ --33E:r0   r   r   r   c                 h    | j                  |||      }| j                  |d   |      }|f|dd  z   }|S )Nr   r   )r   r   )r,   r   r   r   self_outputsattention_outputr   s          r/   r?   zVivitAttention.forward@  sE     ~~mY@QR;;|AF#%QR(88r0   r   )rA   rB   rC   r   r    r   rp   r   rL   ro   r   rE   r   r   r?   rF   rG   s   @r/   r   r   '  s    "{ "t ";S ;d ;* -1"'	|| ELL)  	
 
uU\\5<</0%2EE	Fr0   r   c                   (     e Zd Zdeddf fdZ xZS )VivitSdpaAttentionr-   rU   Nc                 D    t         |   |       t        |      | _        y r   )r   r    r   r   r+   s     r/   r    zVivitSdpaAttention.__init__P  s     /7r0   )rA   rB   rC   r   r    rF   rG   s   @r/   r   r   O  s    8{ 8t 8 8r0   r   c                   $     e Zd Z fdZd Z xZS )VivitIntermediatec                 P   t         |           t        j                  |j                  |j
                        | _        t        j                  |j                        | _	        t        |j                  t              rt        |j                     | _        y |j                  | _        y r   )r   r    r   r{   r&   intermediate_sizer   rQ   rR   rS   
isinstance
hidden_actstrr   intermediate_act_fnr+   s     r/   r    zVivitIntermediate.__init__V  ss    YYv1163K3KL
zz&"<"<=f''-'-f.?.?'@D$'-'8'8D$r0   c                 l    | j                  |      }| j                  |      }| j                  |      }|S r   )r   r   rS   )r,   r   s     r/   r?   zVivitIntermediate.forward_  s4    

=100?]3r0   rA   rB   rC   r    r?   rF   rG   s   @r/   r   r   U  s    9r0   r   c                   $     e Zd Z fdZd Z xZS )VivitOutputc                     t         |           t        j                  |j                  |j
                        | _        t        j                  |j                        | _	        y r   )
r   r    r   r{   r   r&   r   rQ   rR   rS   r+   s     r/   r    zVivitOutput.__init__h  sB    YYv779K9KL
zz&"<"<=r0   c                 T    | j                  |      }| j                  |      }||z   }|S r   r   r   s      r/   r?   zVivitOutput.forwardm  s.    

=1]3%4r0   r   rG   s   @r/   r   r   g  s    >
r0   r   )eagersdpac                   *     e Zd ZdZ fdZddZ xZS )
VivitLayerzNThis corresponds to the EncoderBlock class in the scenic/vivit implementation.c                    t         |           |j                  | _        d| _        t	        |j
                     |      | _        t        |      | _        t        |      | _
        t        j                  |j                  |j                        | _        t        j                  |j                  |j                        | _        y )Nr   eps)r   r    chunk_size_feed_forwardseq_len_dimVIVIT_ATTENTION_CLASSES_attn_implementationr   r   intermediater   r   r   	LayerNormr&   layer_norm_epslayernorm_beforelayernorm_afterr+   s     r/   r    zVivitLayer.__init__  s    '-'E'E$01L1LMfU-f5!&) "V-?-?VEZEZ [!||F,>,>FDYDYZr0   c                     | j                  | j                  |      ||      }|d   }|dd  }||z   }| j                  |      }| j                  |      }| j	                  ||      }|f|z   }|S )N)r   r   r   )r   r   r   r   r   )r,   r   r   r   self_attention_outputsr   r   layer_outputs           r/   r?   zVivitLayer.forward  s    !%!!-0/	 "0 "
 2!4(, )=8 ++M:((6 {{<?/G+r0   r   )rA   rB   rC   rD   r    r?   rF   rG   s   @r/   r   r   }  s    X[r0   r   c                   .     e Zd Z fdZ	 	 	 	 ddZ xZS )VivitEncoderc                     t         |           || _        t        j                  t        |j                        D cg c]  }t        |       c}      | _        d| _	        y c c}w r   )
r   r    r-   r   
ModuleListrangenum_hidden_layersr   layergradient_checkpointing)r,   r-   _r.   s      r/   r    zVivitEncoder.__init__  sN    ]]fF^F^@_#`1Jv$6#`a
&+# $as   A#c                 t   |rdnd }|rdnd }t        | j                        D ]h  \  }}	|r||fz   }|||   nd }
| j                  r+| j                  r| j	                  |	j
                  ||
|      }n
 |	||
|      }|d   }|s`||d   fz   }j |r||fz   }|st        d |||fD              S t        |||      S )N r   r   c              3   &   K   | ]	  }||  y wr   r   ).0vs     r/   	<genexpr>z'VivitEncoder.forward.<locals>.<genexpr>  s     mq_`_lms   )last_hidden_stater   
attentions)	enumerater   r   r   _gradient_checkpointing_func__call__tupler   )r,   r   r   r   output_hidden_statesreturn_dictall_hidden_statesall_self_attentionsilayer_modulelayer_head_masklayer_outputss               r/   r?   zVivitEncoder.forward  s     #7BD$5b4(4 	POA|#$58H$H!.7.CilO**t}} $ A A ))!#%	! !-]OM^ _)!,M &9]1=M<O&O#'	P*   1]4D Dm]4EGZ$[mmm++*
 	
r0   )NFFTr   rG   s   @r/   r   r     s    , ")
r0   r   c                   $     e Zd Z fdZd Z xZS )VivitPoolerc                     t         |           t        j                  |j                  |j                        | _        t        j                         | _        y r   )r   r    r   r{   r&   r   Tanh
activationr+   s     r/   r    zVivitPooler.__init__  s9    YYv1163E3EF
'')r0   c                 \    |d d df   }| j                  |      }| j                  |      }|S )Nr   )r   r  )r,   r   first_token_tensorpooled_outputs       r/   r?   zVivitPooler.forward  s6     +1a40

#566r0   r   rG   s   @r/   r   r     s    $
r0   r   c                   .    e Zd ZdZeZdZdZdZg Z	dZ
d Zy)VivitPreTrainedModelz
    An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
    models.
    vivitr:   Tc                    t        |t        j                  t        j                  f      rm|j                  j
                  j                  d| j                  j                         |j                  %|j                  j
                  j                          yyt        |t        j                        rz|j                  j
                  j                  d| j                  j                         |j                  2|j                  j
                  |j                     j                          yyt        |t        j                        rJ|j                  j
                  j                          |j                  j
                  j                  d       yt        |t        j                        r2|j
                  j                  d| j                  j                         yy)zInitialize the weightsr   )meanstdNg      ?)r   r   r{   r(   weightdatanormal_r-   initializer_rangerv   zero_	Embeddingpadding_idxr   fill_rK   )r,   modules     r/   _init_weightsz"VivitPreTrainedModel._init_weights  s>   fryy"))45 MM&&CT[[5R5R&S{{&  &&( '-MM&&CT[[5R5R&S!!-""6#5#56<<> .-KK""$MM$$S)-KKSdkk.K.KL .r0   N)rA   rB   rC   rD   r   config_classbase_model_prefixmain_input_namesupports_gradient_checkpointing_no_split_modules_supports_sdpar  r   r0   r/   r  r    s0    
 L$O&*#NMr0   r  aG  
    This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it
    as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and
    behavior.

    Parameters:
        config ([`VivitConfig`]): Model configuration class with all the parameters of the model.
            Initializing with a config file does not load the weights associated with the model, only the
            configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
a  
    Args:
        pixel_values (`torch.FloatTensor` of shape `(batch_size, num_frames, num_channels, height, width)`):
            Pixel values. Pixel values can be obtained using [`VivitImageProcessor`]. See
            [`VivitImageProcessor.preprocess`] for details.

        head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*):
            Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`:

            - 1 indicates the head is **not masked**,
            - 0 indicates the head is **masked**.

        output_attentions (`bool`, *optional*):
            Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
            tensors for more detail.
        output_hidden_states (`bool`, *optional*):
            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
            more detail.
        interpolate_pos_encoding (`bool`, *optional*, `False`):
            Whether to interpolate the pre-trained position encodings.
        return_dict (`bool`, *optional*):
            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
z_The bare ViViT Transformer model outputting raw hidden-states without any specific head on top.c                        e Zd Zd fd	Zd Zd Z ee       ee	e
      	 	 	 	 	 	 ddeej                     deej                     dee   dee   d	ed
ee   deeej                     e	f   fd              Z xZS )
VivitModelc                    t         |   |       || _        t        |      | _        t        |      | _        t        j                  |j                  |j                        | _        |rt        |      nd | _        | j                          y )Nr   )r   r    r-   rI   rT   r   encoderr   r   r&   r   	layernormr   pooler	post_init)r,   r-   add_pooling_layerr.   s      r/   r    zVivitModel.__init__/  si     )&1#F+f&8&8f>S>ST->k&)D 	r0   c                 .    | j                   j                  S r   )rT   rO   )r,   s    r/   get_input_embeddingszVivitModel.get_input_embeddings<  s    ///r0   c                     |j                         D ]7  \  }}| j                  j                  |   j                  j	                  |       9 y)z
        Prunes heads of the model.

        Args:
            heads_to_prune:
                dict of {layer_num: list of heads to prune in this layer}
        N)itemsr  r   r   r   )r,   heads_to_pruner   r   s       r/   _prune_headszVivitModel._prune_heads?  sE     +002 	CLE5LLu%//;;EB	Cr0   output_typer  r:   r   r   r   r1   r   rU   c                    ||n| j                   j                  }||n| j                   j                  }||n| j                   j                  }|t	        d      | j                  || j                   j                        }| j                  ||      }| j                  |||||      }|d   }	| j                  |	      }	| j                  | j                  |	      nd}
|s
|	|
f|dd z   S t        |	|
|j                  |j                        S )a  
        Returns:

        Examples:

        ```python
        >>> import av
        >>> import numpy as np

        >>> from transformers import VivitImageProcessor, VivitModel
        >>> from huggingface_hub import hf_hub_download

        >>> np.random.seed(0)


        >>> def read_video_pyav(container, indices):
        ...     '''
        ...     Decode the video with PyAV decoder.
        ...     Args:
        ...         container (`av.container.input.InputContainer`): PyAV container.
        ...         indices (`List[int]`): List of frame indices to decode.
        ...     Returns:
        ...         result (np.ndarray): np array of decoded frames of shape (num_frames, height, width, 3).
        ...     '''
        ...     frames = []
        ...     container.seek(0)
        ...     start_index = indices[0]
        ...     end_index = indices[-1]
        ...     for i, frame in enumerate(container.decode(video=0)):
        ...         if i > end_index:
        ...             break
        ...         if i >= start_index and i in indices:
        ...             frames.append(frame)
        ...     return np.stack([x.to_ndarray(format="rgb24") for x in frames])


        >>> def sample_frame_indices(clip_len, frame_sample_rate, seg_len):
        ...     '''
        ...     Sample a given number of frame indices from the video.
        ...     Args:
        ...         clip_len (`int`): Total number of frames to sample.
        ...         frame_sample_rate (`int`): Sample every n-th frame.
        ...         seg_len (`int`): Maximum allowed index of sample's last frame.
        ...     Returns:
        ...         indices (`List[int]`): List of sampled frame indices
        ...     '''
        ...     converted_len = int(clip_len * frame_sample_rate)
        ...     end_idx = np.random.randint(converted_len, seg_len)
        ...     start_idx = end_idx - converted_len
        ...     indices = np.linspace(start_idx, end_idx, num=clip_len)
        ...     indices = np.clip(indices, start_idx, end_idx - 1).astype(np.int64)
        ...     return indices


        >>> # video clip consists of 300 frames (10 seconds at 30 FPS)
        >>> file_path = hf_hub_download(
        ...     repo_id="nielsr/video-demo", filename="eating_spaghetti.mp4", repo_type="dataset"
        ... )
        >>> container = av.open(file_path)

        >>> # sample 32 frames
        >>> indices = sample_frame_indices(clip_len=32, frame_sample_rate=1, seg_len=container.streams.video[0].frames)
        >>> video = read_video_pyav(container=container, indices=indices)

        >>> image_processor = VivitImageProcessor.from_pretrained("google/vivit-b-16x2-kinetics400")
        >>> model = VivitModel.from_pretrained("google/vivit-b-16x2-kinetics400")

        >>> # prepare video for the model
        >>> inputs = image_processor(list(video), return_tensors="pt")

        >>> # forward pass
        >>> outputs = model(**inputs)
        >>> last_hidden_states = outputs.last_hidden_state
        >>> list(last_hidden_states.shape)
        [1, 3137, 768]
        ```Nz You have to specify pixel_valuesrl   )r   r   r   r   r   r   )r   pooler_outputr   r   )r-   r   r   use_return_dictr6   get_head_maskr   rT   r  r  r   r   r   r   )r,   r:   r   r   r   r1   r   embedding_outputencoder_outputssequence_outputr  s              r/   r?   zVivitModel.forwardJ  s*   n 2C1N-TXT_T_TqTq$8$D $++JjJj 	 &1%<k$++B]B]?@@&&y$++2O2OP	??<Rj?k,,/!5# ' 
 *!,..98<8OO4UY#]3oab6III)-')77&11	
 	
r0   )T)NNNNFN)rA   rB   rC   r    r$  r(  r   VIVIT_INPUTS_DOCSTRINGr   r   _CONFIG_FOR_DOCr   rL   FloatTensorrE   r   r   r?   rF   rG   s   @r/   r  r  *  s    
0	C ++AB+ETcd 5915,0/3).&*u
u001u
 E--.u
 $D>	u

 'tnu
 #'u
 d^u
 
uU&&')CC	Du
 e Cu
r0   r  a  
    ViViT Transformer model with a video classification head on top (a linear layer on top of the final hidden state of the
[CLS] token) e.g. for Kinetics-400.

    <Tip>

        Note that it's possible to fine-tune ViT on higher resolution images than the ones it has been trained on, by
        setting `interpolate_pos_encoding` to `True` in the forward of the model. This will interpolate the pre-trained
        position embeddings to the higher resolution.

    </Tip>
    c                   
    e Zd Z fdZ ee       eee      	 	 	 	 	 	 	 dde	e
j                     de	e
j                     de	e
j                     de	e   de	e   ded	e	e   d
eee
j                     ef   fd              Z xZS )VivitForVideoClassificationc                 .   t         |   |       |j                  | _        t        |d      | _        |j                  dkD  r*t        j                  |j                  |j                        nt        j                         | _	        | j                          y )NF)r"  r   )r   r    
num_labelsr  r  r   r{   r&   Identity
classifierr!  r+   s     r/   r    z$VivitForVideoClassification.__init__  ss      ++%@
 OUN_N_bcNc"))F$6$68I8IJikititiv 	r0   r)  r:   r   labelsr   r   r1   r   rU   c                    ||n| j                   j                  }| j                  ||||||      }|d   }	| j                  |	dddddf         }
d}|}| j                  dk(  r2t               } ||
j                  d      |j                  d            }n<t               } ||
j                  d| j                        |j                  d            }|s|
f|dd z   }||f|z   S |S t        ||
|j                  |j                        S )a(  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the image classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).

        Returns:

        Examples:

        ```python
        >>> import av
        >>> import numpy as np
        >>> import torch

        >>> from transformers import VivitImageProcessor, VivitForVideoClassification
        >>> from huggingface_hub import hf_hub_download

        >>> np.random.seed(0)


        >>> def read_video_pyav(container, indices):
        ...     '''
        ...     Decode the video with PyAV decoder.
        ...     Args:
        ...         container (`av.container.input.InputContainer`): PyAV container.
        ...         indices (`List[int]`): List of frame indices to decode.
        ...     Returns:
        ...         result (np.ndarray): np array of decoded frames of shape (num_frames, height, width, 3).
        ...     '''
        ...     frames = []
        ...     container.seek(0)
        ...     start_index = indices[0]
        ...     end_index = indices[-1]
        ...     for i, frame in enumerate(container.decode(video=0)):
        ...         if i > end_index:
        ...             break
        ...         if i >= start_index and i in indices:
        ...             frames.append(frame)
        ...     return np.stack([x.to_ndarray(format="rgb24") for x in frames])


        >>> def sample_frame_indices(clip_len, frame_sample_rate, seg_len):
        ...     '''
        ...     Sample a given number of frame indices from the video.
        ...     Args:
        ...         clip_len (`int`): Total number of frames to sample.
        ...         frame_sample_rate (`int`): Sample every n-th frame.
        ...         seg_len (`int`): Maximum allowed index of sample's last frame.
        ...     Returns:
        ...         indices (`List[int]`): List of sampled frame indices
        ...     '''
        ...     converted_len = int(clip_len * frame_sample_rate)
        ...     end_idx = np.random.randint(converted_len, seg_len)
        ...     start_idx = end_idx - converted_len
        ...     indices = np.linspace(start_idx, end_idx, num=clip_len)
        ...     indices = np.clip(indices, start_idx, end_idx - 1).astype(np.int64)
        ...     return indices


        >>> # video clip consists of 300 frames (10 seconds at 30 FPS)
        >>> file_path = hf_hub_download(
        ...     repo_id="nielsr/video-demo", filename="eating_spaghetti.mp4", repo_type="dataset"
        ... )
        >>> container = av.open(file_path)

        >>> # sample 32 frames
        >>> indices = sample_frame_indices(clip_len=32, frame_sample_rate=4, seg_len=container.streams.video[0].frames)
        >>> video = read_video_pyav(container=container, indices=indices)

        >>> image_processor = VivitImageProcessor.from_pretrained("google/vivit-b-16x2-kinetics400")
        >>> model = VivitForVideoClassification.from_pretrained("google/vivit-b-16x2-kinetics400")

        >>> inputs = image_processor(list(video), return_tensors="pt")

        >>> with torch.no_grad():
        ...     outputs = model(**inputs)
        ...     logits = outputs.logits

        >>> # model predicts one of the 400 Kinetics-400 classes
        >>> predicted_label = logits.argmax(-1).item()
        >>> print(model.config.id2label[predicted_label])
        LABEL_116
        ```N)r   r   r   r1   r   r   r   rW   r   )losslogitsr   r   )r-   r-  r  r:  r8  r	   rc   r   r   r   r   )r,   r:   r   r;  r   r   r1   r   r   r1  r>  r=  loss_fctr   s                 r/   r?   z#VivitForVideoClassification.forward  s   @ &1%<k$++B]B]**/!5%=#  
 "!*Aq!9:!#"9BRA+-B @&++b/RY,F)-)9TGf$EvE$!//))	
 	
r0   )NNNNNFN)rA   rB   rC   r    r   r2  r   r   r3  r   rL   r4  
LongTensorrE   r   r   r?   rF   rG   s   @r/   r6  r6    s     
 ++AB+@_ 5915-1,0/3).&*@
u001@
 E--.@
 ))*	@

 $D>@
 'tn@
 #'@
 d^@
 
uU&&')>>	?@
 ` C@
r0   r6  )8rD   r   typingr   r   r   r   rL   torch.utils.checkpointr   torch.nnr   r	   activationsr   modeling_outputsr   r   r   modeling_utilsr   pytorch_utilsr   r   utilsr   r   r   r   r   configuration_vivitr   
get_loggerrA   r   _CHECKPOINT_FOR_DOCr3  Moduler   rI   rr   r   r   r   r   r   r   r   r   r   r   r  VIVIT_START_DOCSTRINGr2  r  r6  r   r0   r/   <module>rN     s     . .    . ! b b - Q  - 
		H	%7 *RYY *ZLbii L`9 9z)#/ )#Zbii &$RYY $P8 8		 $")) "  $ $N0
299 0
f")) M? M@	  2 eS
% S
	S
l  O
"6 O
O
r0   