
    sge                        d Z ddlZddlmZ ddlmZmZmZ ddlZ	ddl
mZ ddlmZmZ ddlmZmZ dd	lmZ dd
lmZmZ ddlmZ  ej2                  e      Ze G d de             Z G d de	j:                  j<                  j>                        Z  G d de	j:                  j<                  j>                        Z! G d de	j:                  j<                  j>                        Z" G d de	j:                  j<                  j>                        Z# G d de	j:                  j<                  j>                        Z$ G d de      Z%y)zOTF IdeficsVision model: a copy of CLIPVisionModel using a simpler config object    N)	dataclass)OptionalTupleUnion   )get_tf_activation)TFBaseModelOutputTFBaseModelOutputWithPooling)TFPreTrainedModel
shape_list)flatten)ModelOutputlogging   )IdeficsVisionConfigc                       e Zd ZU dZdZeej                     ed<   dZ	ej                  ed<   dZ
eeej                        ed<   dZeeej                        ed<   y)TFIdeficsVisionModelOutputa  
    Base class for vision model's outputs that also contains image embeddings of the pooling of the last hidden states.

    Args:
        image_embeds (`tf.Tensor` of shape `(batch_size, output_dim)` *optional* returned when model is initialized with `with_projection=True`):
            The image embeddings obtained by applying the projection layer to the pooler_output.
        last_hidden_state (`tf.Tensor` of shape `(batch_size, sequence_length, hidden_size)`):
            Sequence of hidden-states at the output of the last layer of the model.
        hidden_states (`tuple(tf.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
            Tuple of `tf.Tensor` (one for the output of the embeddings, if the model has an embedding layer, +
            one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`.

            Hidden-states of the model at the output of each layer plus the optional initial embedding outputs.
        attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
            Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
            sequence_length)`.

            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
            heads.
    Nimage_embedslast_hidden_statehidden_states
attentions)__name__
__module____qualname____doc__r   r   tfTensor__annotations__r   r   r   r        X/var/www/html/venv/lib/python3.12/site-packages/transformers/models/idefics/vision_tf.pyr   r   "   s[    * )-L(299%,#'ryy'04M8E")),-4-1Jryy)*1r    r   c                        e Zd Zdef fdZdej                  dededej                  fdZddej                  d	e	dej                  fd
Z
ddZ xZS )TFIdeficsVisionEmbeddingsconfigc           	      2   t        |   d
i | || _        |j                  | _        |j
                  | _        |j                  | _        t        j                  j                  j                  | j                  | j                  | j                  dddd      | _        | j
                  | j                  z  dz  | _        | j                  dz   | _        t        j                  j                  j                  | j                  | j                  d	      | _        y )NFvalidchannels_lastpatch_embedding)filterskernel_sizestridesuse_biaspaddingdata_formatname   r   position_embeddingr/   r   )super__init__r$   hidden_size	embed_dim
image_size
patch_sizer   keraslayersConv2Dr(   num_patchesnum_positions	Embeddingr1   selfr$   kwargs	__class__s      r!   r4   z"TFIdeficsVisionEmbeddings.__init__@   s    "6"++ ++ ++!xx55NNOO'"  6  
 !OOt>1D!--1"$((//";";5I #< #
r    
embeddingsheightwidthreturnc           	      ~   t        |      d   dz
  }| j                  | j                        }t        |      d   dz
  }||k(  r||k(  r|S |d d df   }|d d dd f   }t        |      d   }	|| j                  j                  z  }
|| j                  j                  z  }|
dz   |dz   }}
t        j                  t        |            }t        j                  |dt        |      t        |      |	f      }|
|z  }||z  }t        j                  t        j                  |      d   t        j                        }t        j                  t        j                  |      d   t        j                        }t        j                  ||z  t        j                        }t        j                  ||z  t        j                        }t        j                  j!                  |||gt        j                  j"                  j$                        }t        |
      t        |      d   k7  st        |      t        |      d   k7  r@t'        d	t        |
      t        |      f d
t        |      d   t        |      d   f d      t        j                  |dd|	f      }t        j(                  |t        j*                  d d f   |fd      S )Nr   r   g?r0   )sizemethodzNumber of patches for images (z/) don't match the shape of position embedding ()axis)r   r1   position_idsr$   r8   mathsqrtfloatr   reshapeintcastshapefloat32int32imageresizeResizeMethodBICUBIC
ValueErrorconcatnewaxis)r@   rC   rD   rE   r<   	pos_embedr=   class_pos_embedpatch_pos_embedr6   num_h_patchesnum_w_patchessqrt_num_positionsscale_heightscale_widthoriginal_heightoriginal_width
new_height	new_widths                      r!   interpolate_pos_encodingz2TFIdeficsVisionEmbeddings.interpolate_pos_encodingX   sw    ,Q/!3++D,=,=>	"9-a014-'FeO#AqD/#AqrE*z*2.	$++"8"88!7!77'4s':MC<O}!YYu]';<**_q#>P:QSVWiSjlu6vw$'99#&88''"((?";A">

K/!:1!=rzzJWW_|;RXXF
GGN[8"((C	((//:y"9"((BWBWB_B_ * 

 *_"=b"AA=!Z%@%DD0]1CSEW1W0X Y00:?0KB0OQ[\kQlmoQp0p/qqrt  **_q"i6HIyy/"**a-8/JQRSSr    pixel_valuesrm   c                    t        |t              r|d   }t        j                  |d      }t	        |      \  }}}}|sJ|| j
                  k7  s|| j
                  k7  r,t        d| d| d| j
                   d| j
                   d	      | j                  |      }t        |dd	      }t        j                  | j                  t        j                  t        j                  d d f   |d| j                  g      }t        j                  ||gd
      }	|r|	| j                  |	||      z   }	|	S |	| j                  | j                         z   }	|	S )Nrn   )r   r0   r   r   permzInput image size (*z) doesn't match model (z8). You should try to set `interpolate_pos_encoding=True`r   r0   rN   )
isinstancedictr   	transposer   r7   r^   r(   r   broadcast_toclass_embeddingr`   r6   r_   rm   r1   rP   )
r@   rn   rm   
batch_sizerD   rE   num_channelspatch_embedsclass_embedsrC   s
             r!   callzTFIdeficsVisionEmbeddings.call   sN   
 lD)'7L||L|D2<\2J/
FE<'(ET__,D (% 9)4??*;;su 
 ++L9 |Q2  RZZ!:;j!T^^=\
 YYl;!D
 $#d&C&CJPVX]&^^J  $d&=&=d>O>O&PPJr    c                    | j                   ry d| _         t        j                  | j                  d      t        j                  d d f   | _        | j                  | j                  fd      | _        t        | dd       et        j                  | j                  j                        5  | j                  j                  d d d | j                  j                  g       d d d        t        | dd       Nt        j                  | j                   j                        5  | j                   j                  d        d d d        y y # 1 sw Y   exY w# 1 sw Y   y xY w)NTzself.position_idsr2   rw   )rW   r/   r(   r1   )builtr   ranger=   r`   rP   
add_weightr6   rw   getattr
name_scoper(   r/   buildr$   ry   r1   r@   input_shapes     r!   r   zTFIdeficsVisionEmbeddings.build   s    ::
HHT%7%7>QRSUS]S]_`S`a#dnn5FM^_4*D1=t33889 Y$$**D$dkk>V>V+WXY4-t4@t66;;< 4''--d34 4 AY Y4 4s   )4EEE
EFN)r   r   r   r   r4   r   r   rU   rm   boolr|   r   __classcell__rB   s   @r!   r#   r#   ?   sn    
2 
0%T299 %Tc %TRU %TZ\ZcZc %TN! !d !WYW`W` !F4r    r#   c                       e Zd ZdZ fdZdej                  dedefdZ	 	 	 ddej                  de	ej                     d	e	ej                     d
e	e
   deej                  e	ej                     e	eej                        f   f
dZddZ xZS )TFIdeficsVisionAttentionz=Multi-headed attention from 'Attention Is All You Need' paperc                 d   t        |   d
i | || _        |j                  | _        |j
                  | _        | j                  | j                  z  | _        | j                  | j                  z  | j                  k7  r&t        d| j                   d| j                   d      | j                  dz  | _	        |j                  | _        t        j                  j                  j                  | j                  d      | _        t        j                  j                  j                  | j                  d      | _        t        j                  j                  j                  | j                  d      | _        t        j                  j                  j                  | j                  d	      | _        y )Nz;embed_dim must be divisible by num_heads (got `embed_dim`: z and `num_heads`: z).g      k_projr2   v_projq_projout_projr   )r3   r4   r$   r5   r6   num_attention_heads	num_headshead_dimr^   scaleattention_dropoutdropoutr   r9   r:   Denser   r   r   r   r?   s      r!   r4   z!TFIdeficsVisionAttention.__init__   s<   "6"++33$..8==4>>)T^^;MdnnM] ^NN#2'  ]]D(
//hhoo++DNN+Jhhoo++DNN+Jhhoo++DNN+J--dnn:-Nr    tensorseq_lenbszc           	          t        j                  t        j                  |||| j                  | j                  f      g d      S )Nr   r0   r   r   rp   )r   ru   rT   r   r   )r@   r   r   r   s       r!   _shapezTFIdeficsVisionAttention._shape   s0    ||BJJvWdnndmm/\]dpqqr    r   attention_maskcausal_attention_maskoutput_attentionsrF   c           
      ^   t        |      \  }}}| j                  |      | j                  z  }| j                  | j	                  |      d|      }	| j                  | j                  |      d|      }
|| j                  z  d| j                  f}t        j                  | j                  |||      |      }t        j                  |	|      }	t        j                  |
|      }
t        |	      d   }t        j                  j                  ||	d      }t        j                  j                  t        j                  |      || j                  z  ||gd|| j                  z  ||g dt        j                  |              |}t        |      |d||gk7  rt        d	|d||f dt        |             t        j                  ||| j                  ||f      |z   }t        j                  ||| j                  z  ||f      }|}t        |      |d||gk7  rt        d	|d||f dt        |             t        j                  ||| j                  ||f      |z   }t        j                  ||| j                  z  ||f      }t        j                   j#                  |d
      }|rKt        j                  ||| j                  ||f      }t        j                  ||| j                  z  ||f      }nd}t        j                   j%                  || j$                        }t        j                  j                  ||
      }t        j                  j                  t        j                  |      || j                  z  || j                  gd|| j                  z  || j                  g dt        j                  |              t        j                  ||| j                  || j                  f      }t        j&                  |g d      }t        j                  ||||f      }| j)                  |      }||fS )z#Input shape: Batch x Time x ChannelrH   r   T)transpose_bz$Attention weights should be of size z	, but is )messageNz!Attention mask should be of size rN   )rater   rp   )r   r   r   r   r   r   r   r   r   rT   linalgmatmul	debuggingassert_equalrW   r^   nnsoftmaxr   ru   r   )r@   r   r   r   r   r   tgt_lenr6   query_states
key_statesvalue_states
proj_shapesrc_lenattn_weightsattn_weights_reshaped
attn_probsattn_outputs                    r!   r|   zTFIdeficsVisionAttention.call   s    #-]";Wi {{=1DJJ>[[]!;RE
{{4;;}#=r3GDNN*B>
zz$++lGS"I:VZZ
J7
zz,
;Z(+yy''jd'S
!!HH\"4>>!7G4:C$..<PRY[b;c:ddmnpnvnv  xD  oE  nF  G 	" 	
 !,/0S!Wg4NN 7a'8R7S T"#89:<  ::lS$..'SZ4[\_ttL::lS4>>5I7T[4\]L%.)c1gw-GG 7a'8R7SS\]ghv]w\xy  ::lS$..'SZ4[\_mmL::lS4>>5I7T[4\]Luu}}\};
 %'JJ|c4>>SZ\c=d$e!::&;cDNN>RT[]d=efL$(!UU]]<dll]C
ii&&z<@
!!HH[!4>>!7DMM::C$..<PRY[_[h[h;i:jjstvt|t|  ~I  uJ  tK  L 	" 	
 jjsDNNGT]].[\ll;\BjjsGY.GHmmK0111r    c                    | j                   ry d| _         t        | dd       ct        j                  | j                  j
                        5  | j                  j                  | j                  | j                  f       d d d        t        | dd       ct        j                  | j                  j
                        5  | j                  j                  | j                  | j                  f       d d d        t        | dd       ct        j                  | j                  j
                        5  | j                  j                  | j                  | j                  f       d d d        t        | dd       dt        j                  | j                  j
                        5  | j                  j                  | j                  | j                  f       d d d        y y # 1 sw Y   \xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   y xY w)NTr   r   r   r   )r~   r   r   r   r   r/   r   r6   r   r   r   r   s     r!   r   zTFIdeficsVisionAttention.build  s   ::
44(4t{{//0 D!!4>>4>>"BCD44(4t{{//0 D!!4>>4>>"BCD44(4t{{//0 D!!4>>4>>"BCD4T*6t}}112 F##T^^T^^$DEF F 7D DD DD DF Fs0   2G;2G$+2G02G<G!$G-0G9<H)NNFr   )r   r   r   r   r4   r   r   rU   r   r   r   r   r|   r   r   r   s   @r!   r   r      s    GO&rRYY r r3 r /359,1L2yyL2 !+L2  (		2	L2
 $D>L2 
ryy(299-xbii8H/II	JL2\Fr    r   c                   ^     e Zd Z fdZdej
                  dej
                  fdZddZ xZS )TFIdeficsVisionMLPc                 N   t        |   di | || _        t        |j                        | _        t        j                  j                  j                  |j                  d      | _        t        j                  j                  j                  |j                  d      | _        y )Nfc1r2   fc2r   )r3   r4   r$   r   
hidden_actactivation_fnr   r9   r:   r   intermediate_sizer   r5   r   r?   s      r!   r4   zTFIdeficsVisionMLP.__init__*  sw    "6".v/@/@A88??(()A)A(N88??((););%(Hr    r   rF   c                 l    | j                  |      }| j                  |      }| j                  |      }|S r   )r   r   r   )r@   r   s     r!   r|   zTFIdeficsVisionMLP.call1  s4    /**=9/r    c                    | j                   ry d| _         t        | dd       at        j                  | j                  j
                        5  | j                  j                  | j                  j                         d d d        t        | dd       bt        j                  | j                  j
                        5  | j                  j                  | j                  j                         d d d        y y # 1 sw Y   yxY w# 1 sw Y   y xY w)NTr   r   )r~   r   r   r   r   r/   r   r$   r5   r   r   r   s     r!   r   zTFIdeficsVisionMLP.build7  s    ::
4%1txx}}- 8t{{66784%1txx}}- >t{{<<=> > 28 8> >s   0C390C?3C<?Dr   )	r   r   r   r4   r   r   r|   r   r   r   s   @r!   r   r   )  s)    I")) 		 	>r    r   c                        e Zd Zdef fdZ	 d
dej                  dej                  dej                  dee   de	ej                     f
dZ
dd	Z xZS )TFIdeficsVisionEncoderLayerr$   c                 v   t        |   di | |j                  | _        t	        |d      | _        t        j                  j                  j                  |j                  d      | _        t        |d      | _        t        j                  j                  j                  |j                  d      | _        y )N	self_attnr2   layer_norm1epsilonr/   mlplayer_norm2r   )r3   r4   r5   r6   r   r   r   r9   r:   LayerNormalizationlayer_norm_epsr   r   r   r   r?   s      r!   r4   z$TFIdeficsVisionEncoderLayer.__init__D  s    "6"++1&{K88??==fF[F[bo=p%f5988??==fF[F[bo=pr    r   r   r   r   rF   c                     |}| j                  |      }| j                  ||||      \  }}||z   }|}| j                  |      }| j                  |      }||z   }|f}|r||fz  }|S )a9  
        Args:
            hidden_states (`tf.Tensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
            attention_mask (`tf.Tensor`): attention mask of size
                `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values.
                `(config.encoder_attention_heads,)`.
            output_attentions (`bool`, *optional*):
                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
                returned tensors for more detail.
        )r   r   r   r   )r   r   r   r   )r@   r   r   r   r   residualr   outputss           r!   r|   z TFIdeficsVisionEncoderLayer.callL  s    " !((7&*nn')"7/	 '5 '
#| !=0 ((7/ =0 "&Gr    c                    | j                   ry d| _         t        | dd       Zt        j                  | j                  j
                        5  | j                  j                  d d | j                  g       d d d        t        | dd       [t        j                  | j                  j
                        5  | j                  j                  d d | j                  g       d d d        y y # 1 sw Y   rxY w# 1 sw Y   y xY w)NTr   r   )	r~   r   r   r   r   r/   r   r6   r   r   s     r!   r   z!TFIdeficsVisionEncoderLayer.buildt  s    ::
4-9t//445 E  &&dDNN'CDE4-9t//445 E  &&dDNN'CDE E :E EE Es   )C%2)C1%C.1C:r   r   )r   r   r   r   r4   r   r   r   r   r   r|   r   r   r   s   @r!   r   r   C  sl    q2 q -2&yy& 		&  "yy	&
 $D>& 
ryy	&P	Er    r   c                        e Zd ZdZdef fdZ	 	 	 	 	 	 ddeej                     deej                     dee	   dee	   dee	   d	ee	   d
e
eef   fdZddZ xZS )TFIdeficsVisionEncoderz
    Transformer encoder consisting of `config.num_hidden_layers` self attention layers. Each layer is a
    [`TFIdeficsVisionEncoderLayer`].

    Args:
        config: IdeficsVisionConfig
    r$   c                     t        |   di | || _        t        |j                        D cg c]  }t        |d|        c}| _        d| _        y c c}w )Nzlayers.r2   Fr   )r3   r4   r$   r   num_hidden_layersr   r:   gradient_checkpointing)r@   r$   rA   irB   s       r!   r4   zTFIdeficsVisionEncoder.__init__  sZ    "6"MRSYSkSkMl
HI'wqc]C
 ',#
s   Ar   r   r   output_hidden_statesreturn_dicttrainingrF   c                    n| j                   j                  ||n| j                   j                  }||n| j                   j                  }|rdnd}rdnd}	|}
t	        | j
                        D ]\  \  }}|r||
fz   }| j                  r&|r$fd}t        j                   ||      |
||      }n ||
||      }|d   }
sT|	|d   fz   }	^ |r||
fz   }|st        d |
||	fD              S t        |
||	      S )	a  
        Args:
            inputs_embeds (`tf.Tensor` of shape `(batch_size, sequence_length, hidden_size)`):
                Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
                This is useful if you want more control over how to convert `input_ids` indices into associated vectors
                than the model's internal embedding lookup matrix.
            attention_mask (`tf.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
                Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:

                - 1 for tokens that are **not masked**,
                - 0 for tokens that are **masked**.

                [What are attention masks?](../glossary#attention-mask)
            causal_attention_mask (`tf.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
                Causal mask for the text model. Mask values selected in `[0, 1]`:

                - 1 for tokens that are **not masked**,
                - 0 for tokens that are **masked**.

                [What are attention masks?](../glossary#attention-mask)
            output_attentions (`bool`, *optional*):
                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
                returned tensors for more detail.
            output_hidden_states (`bool`, *optional*):
                Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors
                for more detail.
            return_dict (`bool`, *optional*):
                Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
        Nr   c                       fd}|S )Nc                       g |  S r   r   )inputsmoduler   s    r!   custom_forwardzRTFIdeficsVisionEncoder.call.<locals>.create_custom_forward.<locals>.custom_forward  s    %AvA/@AAr    r   )r   r   r   s   ` r!   create_custom_forwardz:TFIdeficsVisionEncoder.call.<locals>.create_custom_forward  s    B *)r    )r   r   r   c              3   &   K   | ]	  }||  y wr   r   ).0vs     r!   	<genexpr>z.TFIdeficsVisionEncoder.call.<locals>.<genexpr>  s     eqWXWdes   )r   r   r   )r$   r   r   use_return_dict	enumerater:   r   r   recompute_gradtupler	   )r@   inputs_embedsr   r   r   r   r   r   encoder_statesall_attentionsr   idxencoder_layerr   layer_outputss       `          r!   r|   zTFIdeficsVisionEncoder.call  sH   N 2C1N-TXT_T_TqTq$8$D $++JjJj 	 &1%<k$++B]B]30d%"+DKK"8 	FC#!/=2B!B**x* !# 1 1)-8!")	! !.!")&7	! *!,M !/=3C2E!E9	F<  +}.>>Ne]NN$Seee +>Vd
 	
r    c                     | j                   ry d| _         t        | dd       K| j                  D ];  }t        j                  |j
                        5  |j                  d        d d d        = y y # 1 sw Y   IxY w)NTr:   )r~   r   r:   r   r   r/   r   )r@   r   layers      r!   r   zTFIdeficsVisionEncoder.build  sp    ::
44(4 &]]5::. &KK%& && 5& &s   A..A7	)NNNNNNr   )r   r   r   r   r   r4   r   r   r   r   r   r   r	   r|   r   r   r   s   @r!   r   r     s    ,2 , /359,0/3&*#'V
 !+V
  (		2	V

 $D>V
 'tnV
 d^V
 4.V
 
u''	(V
p&r    r   c                        e Zd Zdef fdZ	 	 	 	 	 	 ddeej                     dee   dee   dee   dee   dee   d	e	e
ef   fd
ZddZ xZS )TFIdeficsVisionTransformerr$   c                    t        |   |fi | || _        |j                  | _        t        |d      | _        t        j                  j                  j                  |j                  d      | _        t        |d      | _        t        j                  j                  j                  |j                  d      | _        y )NrC   r2   pre_layrnormr   encoderpost_layernorm)r3   r4   r$   r5   r6   r#   rC   r   r9   r:   r   r   r   r   r   r   r?   s      r!   r4   z#TFIdeficsVisionTransformer.__init__  s    *6*++3FNHHOO>>vG\G\cq>r-f9E hhoo@@I^I^eu@vr    rn   r   r   rm   r   r   rF   c                    ||n| j                   j                  }||n| j                   j                  }||n| j                   j                  }|t	        d      | j                  ||      }| j                  |      }| j                  |||||      }|d   }	|	dddddf   }
| j                  |
      }
|s
|	|
f|dd z   S t        |	|
|j                  |j                        S )z
        Returns:

        Nz You have to specify pixel_values)rm   )r   r   r   r   r   r   r   )r   pooler_outputr   r   )r$   r   r   r   r^   rC   r   r   r   r
   r   r   )r@   rn   r   r   rm   r   r   r   encoder_outputsr   pooled_outputs              r!   r|   zTFIdeficsVisionTransformer.call  s    2C1N-TXT_T_TqTq$8$D $++JjJj 	 &1%<k$++B]B]?@@Ogh))-8,,'/!5# ' 
 ,A.)!Q'2++M:%}58KKK+/')77&11	
 	
r    c                    | j                   ry d| _         t        | dd       Mt        j                  | j                  j
                        5  | j                  j                  d        d d d        t        | dd       Zt        j                  | j                  j
                        5  | j                  j                  d d | j                  g       d d d        t        | dd       Mt        j                  | j                  j
                        5  | j                  j                  d        d d d        t        | dd       Zt        j                  | j                  j
                        5  | j                  j                  d | j                  g       d d d        y y # 1 sw Y   3xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   y xY w)NTrC   r   r   r   )r~   r   r   r   rC   r/   r   r   r6   r   r   r   s     r!   r   z TFIdeficsVisionTransformer.build-  ss   ::
4t,8t334 ,%%d+,4.:t00556 F!!''tT^^(DEF4D)5t||001 )""4()4)40<t22778 B##))4*@AB B =, ,F F) )B Bs0   F%)F%F1&(F=F"%F.1F:=G)NNNFNFr   )r   r   r   r   r4   r   r   r   r   r   r   r
   r|   r   r   r   s   @r!   r   r     s    w2 w -1,0/338&*#(,
ryy),
 $D>,
 'tn	,

 #+4.,
 d^,
 4.,
 
u22	3,
\Br    r   )&r   rQ   dataclassesr   typingr   r   r   
tensorflowr   activations_tfr   modeling_tf_outputsr	   r
   modeling_tf_utilsr   r   tf_utilsr   utilsr   r   configuration_ideficsr   
get_loggerr   loggerr   r9   r:   Layerr#   r   r   r   r   r   r   r    r!   <module>r     s    V  ! ) )  / R >  ) 6 
		H	% 2 2 28n4 5 5 n4bvFrxx44 vFr>.. >4:E"((//"7"7 :Ezp&RXX__22 p&fIB!2 IBr    