sg dZddlZddlmZddlmZmZmZm Z ddl Z ddl Z ddl Z ddl mZddlmZddlmZmZdd lmZmZdd lmZdd lmZmZmZmZmZmZd d l m!Z!m"Z"m#Z#ejHe%Z&dZ'de jPde jPfdZ)de jPde jPfdZ*de jPde+fdZ,dGde jPde-de.de+de jPf dZ/dHdZ0dZ1GddejdZ3GddejdZ4Gdd ejdZ5eGd!d"eZ6Gd#d$ejdZ7Gd%d&ejdZ8Gd'd(ejdZ9Gd)d*ejdZ:Gd+d,ejdZ;Gd-d.e;Z<Gd/d0ejdZ=Gd1d2ejdZ>Gd3d4eZ?d5Z@d6ZAd7ZBd8ZCGd9d:ejdZDGd;dejdZFGd?d@e?ZGGdAdBejdZHGdCdDe?ZIee@GdEdFe?ZJy)IzPyTorch GroupViT model.N) dataclass)AnyOptionalTupleUnion)nn)ACT2FN) _create_4d_causal_attention_mask_prepare_4d_attention_mask)BaseModelOutputBaseModelOutputWithPooling)PreTrainedModel) ModelOutputadd_start_docstrings%add_start_docstrings_to_model_forwardloggingreplace_return_docstrings torch_int)GroupViTConfigGroupViTTextConfigGroupViTVisionConfigznvidia/groupvit-gcc-yfcclogitsreturnctjj|tjt ||j S)Ndevice)r functional cross_entropytorcharangelenr)rs a/var/www/html/venv/lib/python3.12/site-packages/transformers/models/groupvit/modeling_groupvit.pycontrastive_lossr%0s/ == & &vu||CKPVP]P]/^ __ similaritycZt|}t|j}||zdz S)Ng@)r%t)r' caption_loss image_losss r$ groupvit_lossr,5s,#J/L!*,,.1J : % ,,r&dimc|j|}|j|dd}tj|tjj ||d}||j z |z}|S)NTkeepdimr memory_format?)softmaxmaxr! zeros_likelegacy_contiguous_formatscatter_detach)rr-y_softindexy_hardrets r$ hard_softmaxr>;sk ^^C F JJsDJ )! ,E   fE4R4R S \ \]`bgil mF 6==? "V +C Jr&tauhardc:tjjjtjd|j |j tjd|j |j }|j|j}||z|z }|j|}|rd|j|dd}tj|tjj||d}||jz |z} | S|} | S)N)rdtyper3Tr/rr1)r! distributionsgumbelGumbeltensorrrCsampleshaper4r5r6r7r8r9) rr?r@r- gumbel_distgumbelsr:r;r<r=s r$gumbel_softmaxrLEs%%,,33 SfllC SfllCK  .G3&G __S !F  3 -a0!!&8V8VW``adfkmpqv}}&/ J Jr&c||z|jdzdz}||kDr4ttj||z }|jd|z}n3ttj||z }|jd|z}|jd}|jd}|j ||||}t j j|||fd|}|S)a Args: attentions (`torch.Tensor`): attention map of shape [batch_size, groups, feat_height*feat_width] height (`int`): height of the output attention map width (`int`): width of the output attention map align_corners (`bool`, *optional*): the `align_corner` argument for `nn.functional.interpolate`. Returns: `torch.Tensor`: resized attention map of shape [batch_size, groups, height, width] ?rrbilinearsizemode align_corners)rIintnproundreshaperr interpolate) attentionsheightwidthrTscale feat_width feat_height batch_sizegroupss r$resize_attention_maprb[se^z//2 2s :E ~%%-01  &&q)Z7 "((6E>23 %%a(K7 !!!$J   a F##J ZPJ**&%z+J r&c Hg}tj5d}|D]i}|jdddj}||}n||z}t |jdddjg|}|j |k ddd|d}|S#1swYxYw)a1 Args: attentions (`tuple(torch.FloatTensor)`: tuple of attention maps returned by `GroupViTVisionTransformer` hw_shape (`tuple(int)`): height and width of the output attention map Returns: `torch.Tensor`: the attention map of shape [batch_size, groups, height, width] NrrNr)r!no_gradpermute contiguousrbappend)rZhw_shape attn_mapsprev_attn_masks attn_masks cur_attn_mapfinal_groupings r$get_grouping_from_attentionsroysI  +$ +J#++Aq!4??AJ&","1J">/0G0G1a0P0[0[0]i`hiL   \ * + +r]N ! + +s A1BB!c*eZdZdeffd ZdZxZS)GroupViTCrossAttentionLayerconfigc"t|t||_t j |j |j|_t||_ t j |j |j|_ yNeps) super__init__GroupViTAttentionattnr LayerNorm hidden_sizelayer_norm_epsnorm2 GroupViTMLPmlp norm_postselfrr __class__s r$rxz$GroupViTCrossAttentionLayer.__init__sb %f- \\&"4"4&:O:OP v&f&8&8f>S>STr&c|}||j||dz}||j|j|z}|j|}|S)N)encoder_hidden_statesr)rzrr~r)rquerykeyxs r$forwardz#GroupViTCrossAttentionLayer.forwardsQ   %s ;A> > A' ' NN1 r&)__name__ __module__ __qualname__rrxr __classcell__rs@r$rqrqsU3Ur&rqc2eZdZdeffd ZddZdZxZS)GroupViTAssignAttentionrrct||jdz|_t j |j|j|_t j |j|j|_t j |j|j|_t j |j|j|_ |j|_ y)N) rwrxr|r]rLinearq_projk_projv_projproj assign_epsrs r$rxz GroupViTAssignAttention.__init__s ''- ii 2 2F4F4FG ii 2 2F4F4FG ii 2 2F4F4FG IIf00&2D2DE  ++r&c|r|jrt|d|}|S|rt|d}|Stjj |d}|S)N)r-r@r-)trainingrLr>rrr4)rrzrEr@s r$get_attnz GroupViTAssignAttention.get_attnsX dmm!$BT:D #Db1 }},,Tr,: r&ct|}|j|}|j|}|j|}||jddz|jz}|j |}|j |dd}||j dd|jzz }||z}|j|}||fS)NrrdF)rEr@Tr-r0) rrr transposer]rsumrr)rrrvalueraw_attnrz soft_attnouts r$rzGroupViTAssignAttention.forwards E"kk# E"CMM"b11TZZ?}}X&MM(5uME txxBx5GHUliinI~r&)TT)rrrrrxrrrrs@r$rrs,3, r&rc0eZdZdeffd ZdZdZxZS)GroupViTTokenAssignrrcdt|||_tj|j |j |_t|jtjjr |jn|j|jf}|Dcgc]}t||j zc}\}}t|||||_tj|j |j |_tj|j |j |_t%||_t)||_tj|j |j |_t/||j ||j |_ycc}wrt)rwrxnum_output_grouprr{r|r} norm_tokens isinstanceassign_mlp_ratio collectionsabcIterablerUGroupViTMixerMLP mlp_internorm_post_tokensnorm_xrqpre_assign_attnrassign norm_new_xr mlp_channels) rrrnum_group_tokenrrr tokens_dim channels_dimrs r$rxzGroupViTTokenAssign.__init__sK  0<<(:(:@U@UV&11;??3K3KL  # #))6+B+BC  JZ#ZACF,>,>(>$?#Z L)&/:O_` " V-?-?VEZEZ [ll6#5#56;P;PQ :6B-f5 ,,v'9'9v?T?TU'0B0BLRXRdRde$[sF-cJ|j|}|j|}|S)z Args: group_tokens (torch.Tensor): group tokens, [batch_size, num_group_tokens, channels] Returns: projected_group_tokens (torch.Tensor): [batch_size, num_output_groups, channels] )rr)r group_tokensprojected_group_tokenss r$project_group_tokenz'GroupViTTokenAssign.project_group_tokens+"& !=!%!6!67M!N%%r&c|j|}|j|}|j|}|j||}|j ||\}}||z }||j |j |z}||fS)z Args: image_tokens (`torch.Tensor`): image tokens, of shape [batch_size, input_length, channels] group_tokens (`torch.Tensor`): group tokens, [batch_size, num_group_tokens, channels] )rrrrrrr)r image_tokensrrnew_image_tokens attentions r$rzGroupViTTokenAssign.forwards'' 5 {{<0 !%!9!9,!G!%!5!56Ll![&*kk2H,&W#)22+d.?.?P`@a.bb**r&)rrrrrxrrrrs@r$rrsf3f* &+r&rceZdZUdZdZeejed<dZ ejed<dZ ejed<dZ ejed<dZ ejed<dZ ejed<dZeed <dZeed <d eefd Zy) GroupViTModelOutputa\ Args: loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `return_loss` is `True`): Contrastive loss for image-text similarity. logits_per_image (`torch.FloatTensor` of shape `(image_batch_size, text_batch_size)`): The scaled dot product scores between `image_embeds` and `text_embeds`. This represents the image-text similarity scores. logits_per_text (`torch.FloatTensor` of shape `(text_batch_size, image_batch_size)`): The scaled dot product scores between `text_embeds` and `image_embeds`. This represents the text-image similarity scores. segmentation_logits (`torch.FloatTensor` of shape `(batch_size, config.num_labels, logits_height, logits_width)`): Classification scores for each pixel. The logits returned do not necessarily have the same size as the `pixel_values` passed as inputs. This is to avoid doing two interpolations and lose some quality when a user needs to resize the logits to the original image size as post-processing. You should always check your logits shape and resize as needed. text_embeds (`torch.FloatTensor` of shape `(batch_size, output_dim`): The text embeddings obtained by applying the projection layer to the pooled output of [`GroupViTTextModel`]. image_embeds (`torch.FloatTensor` of shape `(batch_size, output_dim`): The image embeddings obtained by applying the projection layer to the pooled output of [`GroupViTVisionModel`]. text_model_output (`BaseModelOutputWithPooling`): The output of the [`GroupViTTextModel`]. vision_model_output (`BaseModelOutputWithPooling`): The output of the [`GroupViTVisionModel`]. Nlosslogits_per_imagelogits_per_textsegmentation_logits text_embeds image_embedstext_model_outputvision_model_outputrcHtfdjDS)Nc3dK|]'}|dvr|nt|j)yw))rrN)getattrto_tuple).0krs r$ z/GroupViTModelOutput.to_tuple..:s=  LLDGRYZ^`aRbRkRkRm m s-0)tuplekeysrs`r$rzGroupViTModelOutput.to_tuple9s# YY[   r&)rrr__doc__rrr! FloatTensor__annotations__rrrrrrrrrrrr&r$rr sB)-D(5$$ %,*.e''.)-OU&&--1**1%)K"")&*L%##*48186:3: %* r&rc eZdZdZ d dedeeeeeffdedeffd Zd dejde d ejfd Z xZ S) GroupViTPatchEmbeddingsz# Image to Patch Embedding. image_size patch_size num_channels embed_dimc^t|t|tjj r|n||f}t|tjj r|n||f}|d|dz|d|dzz}||_||_||_tj|||||_ y)Nrr) kernel_sizestride) rwrxrrrrrr num_patchesrConv2d projection)rrrrrrrs r$rxz GroupViTPatchEmbeddings.__init__Es #-j+//:R:R#SZZdfpYq #-j+//:R:R#SZZdfpYq !!} 1 5*Q-:VW=:XY $$&))L)\fgr& pixel_valuesinterpolate_pos_encodingrc 8|j\}}}}|sV||jdk7s||jdk7r2td|d|d|jdd|jdd |j|j dj dd}|S)NrrzInput image size (*z) doesn't match model ().rN)rIr ValueErrorrflattenr)rrrr`rr[r\rs r$rzGroupViTPatchEmbeddings.forwardVs2>2D2D/ L&%'++u8J/J (%9+,Adooa.@-AE OOL ) 1 1! 4 > >q! Dr&)r iF) rrrrrUrrrxr!Tensorboolrrrs@r$rr@s} 24 hh#uS#X./h h  h" ELL D ]b]i]i r&rceZdZdeffd Zdej dededej fdZd dej d e dej fd Z xZ S) GroupViTVisionEmbeddingsrrct|t|j|j|j |j |_|jj}tjtjd||j |_ tj|j|_tj |j |j"|_|j|_||_y)N)rrrrrru)rwrxrrrrr|patch_embeddingsrr Parameterr!zerosposition_embeddingsDropoutdropoutr{r} layernormrr)rrrrrs r$rxz!GroupViTVisionEmbeddings.__init__cs  7((((,,(( !  ++77 #%<< A{FL^L^0_#` zz&..1 f&8&8f>S>ST ++ r& embeddingsr[r\rc0|jd}|jjd}tjj s||k(r||k(r |jS|j}|jd}||j z}||j z} t |dz} |jd| | |}|jdddd}tjj||| fdd }|jddddjdd|}|S) a This method allows to interpolate the pre-trained position encodings, to be able to use the model on higher resolution images. This method is also adapted to support torch.jit tracing and no class embeddings. Adapted from: - https://github.com/facebookresearch/dino/blob/de9ee3df6cf39fac952ab558447af1fa1365362a/vision_transformer.py#L174-L194, and - https://github.com/facebookresearch/dinov2/blob/e1277af2ba9496fbadf7aec6eba56e8d882d1e35/dinov2/models/vision_transformer.py#L179-L211 rrdrOrr rNbicubicFrQ) rIrr!jit is_tracingrrrXrfrrrYview) rrr[r\r num_positionspatch_pos_embedr- new_height new_widthsqrt_num_positionss r$rz1GroupViTVisionEmbeddings.interpolate_pos_encodingss#!&&q) 0066q9 yy##%+*F6UZ?++ +22r"t. T__, &}c'9:)11!5GI[]`a)11!Q1=--33 i( 4 *11!Q1=BB1b#Nr&rrc |j\}}}}|j||}|j|}|j\}}} |r||j |||z}n||j z}|j |}|S)N)r)rIrrrRrrr) rrrr`rr[r\rseq_len_s r$rz GroupViTVisionEmbeddings.forwards2>2D2D/ L&%**&>>J\\*- r&r) rrrrrxr!rrUrrrrrs@r$rrbsc3 $5<<$$UX$]b]i]i$LELLD]b]i]ir&rc eZdZdeffd Z ddeejdeejdeejdejfdZ xZ S) GroupViTTextEmbeddingsrrcNt||j}tj|j ||_tj|j||_|jdtj|jjddy)N position_ids)rrdF) persistent) rwrxr|r Embedding vocab_sizetoken_embeddingmax_position_embeddingsposition_embeddingregister_bufferr!r"expandrrrrrs r$rxzGroupViTTextEmbeddings.__init__s && !||F,=,=yI"$,,v/M/My"Y  ELL)G)GHOOPWXej  r& input_idsr inputs_embedsrc||jdn|jd}||jddd|f}||j|}|j|}||z}|S)Nrdr)rIrrr)rrrr seq_lengthrrs r$rzGroupViTTextEmbeddings.forwards{ -6,AY__R(}GZGZ[]G^  ,,Q  ^ ' u  ! 'r&r c eZdZ d dedeedeedeeffd ZdejdejfdZ xZ S) rrrr|intermediate_size output_sizect|||_t|j|_||n |j }||n |j}||n|}tj|||_ tj|||_ yr-) rwrxrrr hidden_act activation_fnr|rArrfc1fc2)rrrr|rArBrs r$rxzGroupViTMLP.__init__-s  #F$5$56%0%> )T^^ ;MdnnM]^NN#2' ]]D( // ii? ii? ii?  $..$..A r&rGr bszc|j|||j|jjddj SrK)rrOrPrrg)rrGr rSs r$_shapezGroupViTAttention._shape`s7{{3GQQRSUVWbbddr&r4r8r9rr6rc*|j\}}}|du} |j||jz} | rE|j|j |d|} |j|j |d|} nD|j|j |d|} |j|j |d|} ||j zd|jf} |j| ||j| } | j| } | j| } | jd}tj| | jdd}|j||j z||fk7r/td||j z||fd|j|{|j|d||fk7r#td|d||fd|j|j||j |||z}|j||j z||}|{|j|d||fk7r#td|d||fd|j|j||j |||z}|j||j z||}tjj|d}|r?|j||j ||}|j||j z||}nd}tjj!||j |j" }tj|| }|j||j z||jfk7r7td ||j ||jfd|j|j||j ||j}|jdd}|j%|||}|j'|}||fS) z#Input shape: Batch x Time x ChannelNrdrrNz$Attention weights should be of size z , but is z!Attention mask should be of size r)prz `attn_output` should be of size )rRrr]rUrrrOrPrr!bmmrrrrr4rrrXrR)rr4r8r9rr6rStgt_lenris_cross_attention query_states key_states value_states proj_shapesrc_len attn_weightsattn_weights_reshaped attn_probs attn_outputs r$rzGroupViTAttention.forwardcs#0"4"4"6Wi2$>{{=1DJJ> T[[1F%GSQJ;;t{{3H'I2sSLT[[%?SIJ;;t{{='A2sKLDNN*B > Ct{{<#>CCZP $Z__j1 (|((*5 //!$yyz/C/CAq/IJ    3#7'"J J6dnn8LgW^7_6`a %%'(*  ! ,$))+Q/II 7a'8R7ST-22457(,,S$..'7SVkkL',,S4>>-A7GTL  %""$a'(BB 7a'8R7SS\]k]p]p]r\st(,,S$..'7SVddL',,S4>>-A7GTL}},,\r,B  %1$5$5c4>>7T[$\ !055cDNN6JGU\]L$( !]]**<4< R2 u||Xell3XeELL>Q5RR SR2r&ryc eZdZdeffd Z d dej dej dej deede ejf dZ xZ S) r'rrcDt||j|_t ||_t j|j|j|_ t||_ t j|j|j|_ yrt) rwrxr|rry self_attnrr{r} layer_norm1rr layer_norm2rs r$rxzGroupViTEncoderLayer.__init__sm ++*62<<F & u  ! &r&r'c"eZdZdZeZdZdZdZy)GroupViTPreTrainedModelz An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained models. groupvitTcd|jj}t|tjtj frX|j jjd||j|jjjnct|tjrI|jjj|j jjd|jj}t|tri|jj jjd|dz|j j jjd|dzyt|t"r,|jj}|j$dzd|jj&zdzz|z}|j$dz|z}tj(j|j*j |tj(j|j,j |tj(j|j.j |tj(j|j0j |yt|t2r|jj}|jj4dzd|jj&zdzz|z}d|jj4zdz|z}tj(j|j6j |tj(j|j8j |yy) zInitialize the weightsrB)meanstdNr3g{Gz?rrN)rp)rrinitializer_rangerrrrweightdatanormal_biaszero_r{fill_initializer_factorr rrryrnum_hidden_layersinitrrrrRrr|rFrG)rmodule init_rangefactor in_proj_std out_proj_stdfc_stds r$ _init_weightsz%GroupViTPreTrainedModel._init_weightss[[22 fryy"))4 5 MM   & &CZ & @{{&   &&(  - KK   " " $ MM   $ $S )// f4 5  " " ) ) . . 6 6CVd] 6 S  % % , , 1 1 9 9sQU 9 V  1 2[[33F!++T1q6==;Z;Z7Z_c6cdgmmK",,d2f