sgͨdZddlZddlZddlmZddlmZmZm Z m Z ddl Z ddl Z ddl m Z ddlmZmZmZddlmZdd lmZmZmZmZdd lmZdd lmZmZdd lmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%d dl&m'Z'e#jPe)Z*dZ+dZ,gdZ-dZ.dZ/Gdde j`Z1Gdde j`Z2Gdde j`Z3Gdde3Z4Gdde j`Z5Gdde j`Z6Gdd e6Z7Gd!d"e j`Z8Gd#d$e j`Z9e6e7d%Z:Gd&d'e j`Z;Gd(d)e j`Z<Gd*d+eZ=d,Z>d-Z?e!d.e>Gd/d0e=Z@Gd1d2e j`ZAe!d3e>Gd4d5e=ZBe!d6e>Gd7d8e=ZCeGd9d:eZDe!d;e>Gd<d=e=ZEy)>zPyTorch DeiT model.N) dataclass)OptionalSetTupleUnion)nn)BCEWithLogitsLossCrossEntropyLossMSELoss)ACT2FN)BaseModelOutputBaseModelOutputWithPoolingImageClassifierOutputMaskedImageModelingOutput)PreTrainedModel) find_pruneable_heads_and_indicesprune_linear_layer) ModelOutputadd_code_sample_docstringsadd_start_docstrings%add_start_docstrings_to_model_forwardloggingreplace_return_docstrings torch_int) DeiTConfigrz(facebook/deit-base-distilled-patch16-224)riztabby, tabby catc eZdZdZddededdffd Zdejde d e dejfd Z dd ejd e ejd edejfdZ xZS)DeiTEmbeddingszv Construct the CLS token, distillation token, position and patch embeddings. Optionally, also the mask token. configuse_mask_tokenreturnNct|tjt j dd|j |_tjt j dd|j |_|r4tjt j dd|j nd|_ t||_ |jj}tjt j d|dz|j |_ tj|j|_|j"|_y)Nr)super__init__r Parametertorchzeros hidden_size cls_tokendistillation_token mask_tokenDeiTPatchEmbeddingspatch_embeddings num_patchesposition_embeddingsDropouthidden_dropout_probdropout patch_size)selfr!r"r1 __class__s Y/var/www/html/venv/lib/python3.12/site-packages/transformers/models/deit/modeling_deit.pyr'zDeiTEmbeddings.__init__Cs ekk!Q8J8J&KL"$,,u{{1aASAS/T"UQ_",,u{{1a9K9K'LMei 3F ;++77 #%<< A{QPVPbPb0c#d zz&"<"<=  ++ embeddingsheightwidthc|jddz }|jjddz }tjj s||k(r||k(r |jS|jddddf}|jddddf}|jd}||j z} ||j z} t |dz} |jd| | |}|jdddd}tjj|| | fdd }|jddddjdd|}tj||fd S) a This method allows to interpolate the pre-trained position encodings, to be able to use the model on higher resolution images. This method is also adapted to support torch.jit tracing and 2 class embeddings. Adapted from: - https://github.com/facebookresearch/dino/blob/de9ee3df6cf39fac952ab558447af1fa1365362a/vision_transformer.py#L174-L194, and - https://github.com/facebookresearch/dinov2/blob/e1277af2ba9496fbadf7aec6eba56e8d882d1e35/dinov2/models/vision_transformer.py#L179-L211 rr%N?rr bicubicF)sizemode align_cornersdim)shaper2r)jit is_tracingr6rreshapepermuter functional interpolateviewcat) r7r;r<r=r1 num_positionsclass_and_dist_pos_embedpatch_pos_embedrF new_height new_widthsqrt_num_positionss r9interpolate_pos_encodingz'DeiTEmbeddings.interpolate_pos_encodingOsb!&&q)A- 0066q9A= yy##%+*F6UZ?++ +#'#;#;ArrE#B 221ab59r"t. T__, &}c'9:)11!5GI[]`a)11!Q1=--33 i( 4 *11!Q1=BB1b#Nyy2OD!LLr: pixel_valuesbool_masked_posrVc"|j\}}}}|j|}|j\}} }|K|jj || d} |j dj | } |d| z z| | zz}|jj |dd} |jj |dd} tj| | |fd}|j}|r|j|||}||z}|j|}|S)Nr??rrE)rGr0rBr.expand unsqueezetype_asr,r-r)rOr2rVr5)r7rWrXrV_r<r=r; batch_size seq_length mask_tokensmask cls_tokensdistillation_tokensposition_embeddings r9forwardzDeiTEmbeddings.forwardws +001fe**<8 $.OO$5! J  &//00ZLK",,R088ED#sTz2[45GGJ^^**:r2> "55<!>z6SX!Y "44 \\*- r:)FNF)__name__ __module__ __qualname____doc__rboolr'r)TensorintrVr BoolTensorrf __classcell__r8s@r9r r >s ,z ,4 ,D ,&M5<<&M&MUX&M]b]i]i&MV7;). ll"%"2"23#'   r:r cZeZdZdZfdZdej dej fdZxZS)r/z This class turns `pixel_values` of shape `(batch_size, num_channels, height, width)` into the initial `hidden_states` (patch embeddings) of shape `(batch_size, seq_length, hidden_size)` to be consumed by a Transformer. ct||j|j}}|j|j }}t |tjjr|n||f}t |tjjr|n||f}|d|dz|d|dzz}||_||_||_||_ tj|||||_ y)Nrr) kernel_sizestride)r&r' image_sizer6 num_channelsr+ isinstance collectionsabcIterabler1rConv2d projection)r7r!rvr6rwr+r1r8s r9r'zDeiTPatchEmbeddings.__init__s !'!2!2F4E4EJ $*$7$79K9Kk #-j+//:R:R#SZZdfpYq #-j+//:R:R#SZZdfpYq !!} 1 5*Q-:VW=:XY $$(&))L+:^hir:rWr#c|j\}}}}||jk7r td|j|j dj dd}|S)NzeMake sure that the channel dimension of the pixel values match with the one set in the configuration.r%r)rGrw ValueErrorr}flatten transpose)r7rWr_rwr<r=xs r9rfzDeiTPatchEmbeddings.forwardsa2>2D2D/ L&% 4,, ,w  OOL ) 1 1! 4 > >q! Dr:) rhrirjrkr'r)rmrfrprqs@r9r/r/s) jELLU\\r:r/c eZdZdeddffd Zdej dej fdZ d deej de de e ej ej fe ej ffd Z xZ S) DeiTSelfAttentionr!r#Nct||j|jzdk7r3t |ds't d|jfd|jd|j|_t |j|jz |_|j|jz|_tj|j|j|j|_ tj|j|j|j|_ tj|j|j|j|_tj|j |_y)Nrembedding_sizezThe hidden size z4 is not a multiple of the number of attention heads .)bias)r&r'r+num_attention_headshasattrrrnattention_head_size all_head_sizerLinearqkv_biasquerykeyvaluer3attention_probs_dropout_probr5r7r!r8s r9r'zDeiTSelfAttention.__init__s1     : : :a ?PVXhHi"6#5#5#6"78334A7  $*#=#= #&v'9'9FS"S%**+BC 6G=/2O\M]r:rg)rhrirjrr'r)rmrrrlrrrfrprqs@r9rrsGzGdG$%ell%u||% bg!(0(>!Z^! uU\\5<</0% 2EE F!r:rc eZdZdeddffd Z d dej deejde de e ejejfe ejfffd Z xZ S) DeiTSdpaSelfAttentionr!r#NcFt|||j|_yN)r&r'rrs r9r'zDeiTSdpaSelfAttention.__init__s  ,2,O,O)r:rrrc ^|s|'tjdt | |||S|j |}|j |j |}|j |j|}|j |}tjjj|||||jr |jnddd}|jdddd j}|j!dd |j"fz} |j%| }|dfS) Na`DeiTSdpaAttention` is used but `torch.nn.functional.scaled_dot_product_attention` does not support `output_attentions=True` or `head_mask`. Falling back to the manual attention implementation, but specifying the manual implementation will be required from Transformers version v5.0.0 onwards. This warning can be removed using the argument `attn_implementation="eager"` when loading the model.)rrrF) is_causalscalerr%rr r)logger warning_oncer&rfrrrrr)rrLscaled_dot_product_attentiontrainingrrKrrBrrN) r7rrrrrrrrrr8s r9rfzDeiTSdpaSelfAttention.forwards:  5   w  7?+#"3#  !JJ}5--dhh}.EF // =0IJ //0AB ++HH    15D - -CI &--aAq9DDF "/"4"4"6s";t?Q?Q>S"S%**+BC d""r:rg)rhrirjrr'r) FloatTensorrrmrlrrrfrprqs@r9rrsPzPdP-1"' '#(('#ELL)'# '# uU\\5<</0% 2EE F '#'#r:rc|eZdZdZdeddffd ZdejdejdejfdZxZ S) DeiTSelfOutputz The residual connection is defined in DeiTLayer instead of here (as is the case with other models), due to the layernorm applied before each block. r!r#Nct|tj|j|j|_tj |j|_yr) r&r'rrr+denser3r4r5rs r9r'zDeiTSelfOutput.__init__+sB YYv1163E3EF zz&"<"<= r:r input_tensorcJ|j|}|j|}|Srrr5r7rrs r9rfzDeiTSelfOutput.forward0s$ =1  ]3 r:) rhrirjrkrr'r)rmrfrprqs@r9rr%sD >z>d> U\\RWR^R^r:rc eZdZdeddffd ZdeeddfdZ d dejde ejd e de e ejejfe ejffd ZxZS) DeiTAttentionr!r#Nct|t||_t ||_t |_yr)r&r'r attentionroutputset pruned_headsrs r9r'zDeiTAttention.__init__9s0 *62$V, Er:headsc>t|dk(ryt||jj|jj|j \}}t |jj||j_t |jj||j_t |jj||j_ t |jj|d|j_ |jjt|z |j_|jj|jjz|j_ |j j||_y)NrrrE)lenrrrrrrrrrrrrunion)r7rindexs r9 prune_headszDeiTAttention.prune_heads?s u:? 7 4>>55t~~7Y7Y[_[l[l u 2$..2F2FN/0B0BEJ1$..2F2FN.t{{/@/@%QO .2^^-O-ORUV[R\-\*'+~~'I'IDNNLnLn'n$ --33E:r:rrrch|j|||}|j|d|}|f|ddz}|S)Nrr)rr)r7rrr self_outputsattention_outputrs r9rfzDeiTAttention.forwardQsE ~~mY@QR ;;|A F#% QR(88r:rg)rhrirjrr'rrnrr)rmrrlrrrfrprqs@r9rr8s"z"d" ;S;d;*-1"' || ELL)   uU\\5<</0% 2EE F r:rc(eZdZdeddffd ZxZS)DeiTSdpaAttentionr!r#NcDt||t||_yr)r&r'rrrs r9r'zDeiTSdpaAttention.__init__as  .v6r:)rhrirjrr'rprqs@r9rr`s7z7d77r:rc`eZdZdeddffd Zdej dej fdZxZS)DeiTIntermediater!r#Nct|tj|j|j |_t|jtrt|j|_ y|j|_ yr) r&r'rrr+intermediate_sizerrx hidden_actstrr intermediate_act_fnrs r9r'zDeiTIntermediate.__init__hs] YYv1163K3KL f'' -'-f.?.?'@D $'-'8'8D $r:rcJ|j|}|j|}|Sr)rr)r7rs r9rfzDeiTIntermediate.forwardps& =1 00? r: rhrirjrr'r)rmrfrprqs@r9rrgs19z9d9U\\ellr:rcxeZdZdeddffd Zdej dej dej fdZxZS) DeiTOutputr!r#Nct|tj|j|j |_tj|j|_ yr) r&r'rrrr+rr3r4r5rs r9r'zDeiTOutput.__init__ysB YYv779K9KL zz&"<"<= r:rrcT|j|}|j|}||z}|Srrrs r9rfzDeiTOutput.forward~s. =1  ]3 % 4 r:rrqs@r9rrxs?>z>d> U\\RWR^R^r:r)eagersdpac eZdZdZdeddffd Z d dejdeejde de e ejejfe ejffd Z xZ S) DeiTLayerz?This corresponds to the Block class in the timm implementation.r!r#Nct||j|_d|_t |j ||_t||_t||_ tj|j|j|_tj|j|j|_y)Nreps)r&r'chunk_size_feed_forward seq_len_dimDEIT_ATTENTION_CLASSES_attn_implementationrr intermediaterrr LayerNormr+layer_norm_epslayernorm_beforelayernorm_afterrs r9r'zDeiTLayer.__init__s '-'E'E$/0K0KLVT,V4 ( " V-?-?VEZEZ [!||F,>,>FDYDYZr:rrrc|j|j|||}|d}|dd}||z}|j|}|j|}|j ||}|f|z}|S)N)rrr)rrrrr)r7rrrself_attention_outputsrr layer_outputs r9rfzDeiTLayer.forwards "&  ! !- 0 /"0"  2!4(,)=8 ++M: ((6 {{<? /G+r:rg)rhrirjrkrr'r)rmrrlrrrfrprqs@r9rrsI[z[d[-1"' ||ELL)  uU\\5<</0% 2EE F r:rceZdZdeddffd Z d dej deej deded ede e e ff d Z xZ S) DeiTEncoderr!r#Nct|||_tjt |j Dcgc] }t|c}|_d|_ ycc}wrg) r&r'r!r ModuleListrangenum_hidden_layersrlayergradient_checkpointing)r7r!r^r8s r9r'zDeiTEncoder.__init__sN  ]]uVE]E]?^#_!If$5#_` &+#$`sA#rrroutput_hidden_states return_dictct|rdnd}|rdnd}t|jD]h\}} |r||fz}|||nd} |jr+|jr|j | j || |} n | || |} | d}|s`|| dfz}j|r||fz}|st d|||fDSt|||S)Nrrc3&K|] }|| ywrr).0vs r9 z&DeiTEncoder.forward..smq_`_lms)last_hidden_stater attentions) enumeraterrr_gradient_checkpointing_func__call__tupler) r7rrrrrall_hidden_statesall_self_attentionsi layer_modulelayer_head_mask layer_outputss r9rfzDeiTEncoder.forwards#7BD$5b4(4 POA|#$58H$H!.7.CilO**t}} $ A A ))!#% ! !-]OM^ _ )!,M &9]1=M