
    sgq                    0   d Z ddlmZ ddlZddlZddlmZ ddlm	Z	m
Z
mZ ddlZddlmZ ddlmZmZmZmZ dd	lmZmZmZmZmZmZ dd
lmZmZ ddlm Z m!Z!m"Z"m#Z#m$Z$m%Z% ddl&m'Z'  e$jP                  e)      Z*dZ+dZ,g dZ-dZ.dZ/e G d de              Z0 G d dejb                  jd                        Z3 G d dejb                  jd                        Z4 G d dejb                  jd                        Z5 G d dejb                  jd                        Z6 G d dejb                  jd                        Z7 G d dejb                  jd                        Z8 G d  d!ejb                  jd                        Z9 G d" d#ejb                  jd                        Z: G d$ d%ejb                  jd                        Z;e G d& d'ejb                  jd                               Z< G d( d)e      Z=d*Z>d+Z? e"d,e>       G d- d.e=             Z@ G d/ d0ejb                  jd                        ZA G d1 d2ejb                  jd                        ZB G d3 d4ejb                  jd                        ZC e"d5e>       G d6 d7e=             ZD e"d8e>       G d9 d:e=e             ZE e"d;e>       G d< d=e=             ZFy)>zTensorFlow DeiT model.    )annotationsN)	dataclass)OptionalTupleUnion   )get_tf_activation)TFBaseModelOutputTFBaseModelOutputWithPoolingTFImageClassifierOutputTFMaskedImageModelingOutput)TFPreTrainedModelTFSequenceClassificationLossget_initializerkeraskeras_serializableunpack_inputs)
shape_liststable_softmax)ModelOutputadd_code_sample_docstringsadd_start_docstrings%add_start_docstrings_to_model_forwardloggingreplace_return_docstrings   )
DeiTConfigr   z(facebook/deit-base-distilled-patch16-224)r      i   ztabby, tabby catc                  X    e Zd ZU dZdZded<   dZded<   dZded<   dZded<   dZ	ded	<   y)
-TFDeiTForImageClassificationWithTeacherOutputa  
    Output type of [`DeiTForImageClassificationWithTeacher`].

    Args:
        logits (`tf.Tensor` of shape `(batch_size, config.num_labels)`):
            Prediction scores as the average of the cls_logits and distillation logits.
        cls_logits (`tf.Tensor` of shape `(batch_size, config.num_labels)`):
            Prediction scores of the classification head (i.e. the linear layer on top of the final hidden state of the
            class token).
        distillation_logits (`tf.Tensor` of shape `(batch_size, config.num_labels)`):
            Prediction scores of the distillation head (i.e. the linear layer on top of the final hidden state of the
            distillation token).
        hidden_states (`tuple(tf.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
            Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of shape
            `(batch_size, sequence_length, hidden_size)`. Hidden-states of the model at the output of each layer plus
            the initial embedding outputs.
        attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
            Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
            sequence_length)`. Attentions weights after the attention softmax, used to compute the weighted average in
            the self-attention heads.
    N	tf.Tensorlogits
cls_logitsdistillation_logitszTuple[tf.Tensor] | Nonehidden_states
attentions)
__name__
__module____qualname____doc__r"   __annotations__r#   r$   r%   r&        \/var/www/html/venv/lib/python3.12/site-packages/transformers/models/deit/modeling_tf_deit.pyr    r    C   s>    , FI J	 %))-1M*1*.J'.r-   r    c                  X     e Zd ZdZdd fdZddZd	dZ	 	 	 d
	 	 	 	 	 	 	 	 	 ddZ xZS )TFDeiTEmbeddingszv
    Construct the CLS token, distillation token, position and patch embeddings. Optionally, also the mask token.
    c                    t        |   di | || _        || _        t	        |d      | _        t        j                  j                  |j                  d      | _
        y )Npatch_embeddings)confignamedropoutr4   r,   )super__init__r3   use_mask_tokenTFDeiTPatchEmbeddingsr2   r   layersDropouthidden_dropout_probr5   )selfr3   r9   kwargs	__class__s       r.   r8   zTFDeiTEmbeddings.__init__g   sS    "6", 5VJ\ ]||++F,F,FY+Wr-   c                   | j                  dd| j                  j                  ft        j                  j                         dd      | _        | j                  dd| j                  j                  ft        j                  j                         dd      | _        d | _        | j                  rM| j                  dd| j                  j                  ft        j                  j                         dd      | _        | j                  j                  }| j                  d|dz   | j                  j                  ft        j                  j                         dd      | _        | j                  ry d| _        t        | d	d       Mt        j                   | j                  j"                        5  | j                  j%                  d        d d d        t        | d
d       Nt        j                   | j&                  j"                        5  | j&                  j%                  d        d d d        y y # 1 sw Y   exY w# 1 sw Y   y xY w)Nr   T	cls_token)shapeinitializer	trainabler4   distillation_token
mask_token   position_embeddingsr2   r5   )
add_weightr3   hidden_sizer   initializerszerosrB   rF   rG   r9   r2   num_patchesrI   builtgetattrtf
name_scoper4   buildr5   )r>   input_shaperN   s      r.   rS   zTFDeiTEmbeddings.buildn   s   a001**002	 ) 
 #'//a001**002%	 #2 #
 "oo!T[[445!..446!	 . DO ++77#'??kAot{{'>'>?**002&	 $3 $
  ::
4+T2>t4499: 2%%++D124D)5t||001 )""4() ) 62 2) )s   +H+H7+H47I c           
        |j                   d   dz
  }| j                  j                   d   dz
  }||k(  r||k(  r| j                  S | j                  d d dd d f   }| j                  d d dd d f   }| j                  d d dd d d f   }|j                   d   }	|| j                  j                  z  }
|| j                  j                  z  }|
dz   |dz   }}
t	        j
                  |dt        t        j                  |            t        t        j                  |            |	f      }t        j                  j                  |t        |
      t        |      fd      }t	        j                  |g d	      }t	        j
                  |dd|	f      }t	        j                  t	        j                  |d
      t	        j                  |d
      |gd
      S )Nr   rH   r   g?bicubic)sizemethodr   rH   r   r   permaxis)rC   rI   r3   
patch_sizerQ   reshapeintmathsqrtimageresize	transposeconcatexpand_dims)r>   
embeddingsheightwidthrN   num_positionsclass_pos_embeddist_pos_embedpatch_pos_embeddimh0w0s               r.   interpolate_pos_encodingz)TFDeiTEmbeddings.interpolate_pos_encoding   s    &&q)A-0066q9A=-'FeO+++221a7;11!Q':221ab!8<r"t{{---dkk,,, c28B**aTYY}%=!>DIImD\@]_bc
 ((///R#b'@R[d/e,,\J**_q"clCyy^^O!4bnn^Z[6\^mnuv
 	
r-   c                l   |j                   \  }}}}| j                  |      }t        |      \  }	}
}|it        j                  | j
                  |	|
dg      }t        j                  |d      }t        j                  ||j                        }|d|z
  z  ||z  z   }t        j                  | j                  |	d      }t        j                  | j                  |	d      }t        j                  |||fd      }| j                  }|r| j                  |||      }||z   }| j                  ||      }|S )	Nr   rV   r]   dtypeg      ?r   )repeatsr^   training)rC   r2   r   rQ   tilerG   rh   castrv   repeatrB   rF   rg   rI   rs   r5   )r>   pixel_valuesbool_masked_posry   rs   _rj   rk   ri   
batch_size
seq_lengthmask_tokensmask
cls_tokensdistillation_tokensposition_embeddings                   r.   callzTFDeiTEmbeddings.call   s)    +0065!**<8
$.z$:!
J&''$//J
A3NOK>>/;D774{'8'89D#sTz2[45GGJYYt~~zJ
 ii(?(?Z[\YY
,?LSTU
!55#!%!>!>z6SX!Y"44
\\*x\@
r-   F)r3   r   r9   boolreturnNoneN)ri   r!   rj   ra   rk   ra   r   r!   )NFF)
r}   r!   r~   tf.Tensor | Nonery   r   rs   r   r   r!   )	r'   r(   r)   r*   r8   rS   rs   r   __classcell__r@   s   @r.   r0   r0   b   sY    X%)N
< -1). * 	
 #' 
r-   r0   c                  4     e Zd ZdZd fdZddZddZ xZS )r:   z
    This class turns `pixel_values` of shape `(batch_size, num_channels, height, width)` into the initial
    `hidden_states` (patch embeddings) of shape `(batch_size, seq_length, hidden_size)` to be consumed by a
    Transformer.
    c                   t        |   di | |j                  |j                  }}|j                  |j
                  }}t        |t        j                  j                        r|n||f}t        |t        j                  j                        r|n||f}|d   |d   z  |d   |d   z  z  }|| _        || _        || _        || _
        t        j                  j                  |||d      | _        y )Nr   r   
projection)kernel_sizestridesr4   r,   )r7   r8   
image_sizer_   num_channelsrK   
isinstancecollectionsabcIterablerN   r   r;   Conv2Dr   )	r>   r3   r?   r   r_   r   rK   rN   r@   s	           r.   r8   zTFDeiTPatchEmbeddings.__init__   s    "6"!'!2!2F4E4EJ
$*$7$79K9Kk#-j+//:R:R#SZZdfpYq
#-j+//:R:R#SZZdfpYq
!!}
15*Q-:VW=:XY$$(&,,--Z, . 
r-   c                    t        |      \  }}}}t        j                         r|| j                  k7  rt	        d      | j                  |      }t        |      \  }}}}t        j                  ||||z  |f      }|S )NzeMake sure that the channel dimension of the pixel values match with the one set in the configuration.)r   rQ   executing_eagerlyr   
ValueErrorr   r`   )r>   r}   r   rj   rk   r   xs          r.   r   zTFDeiTPatchEmbeddings.call   s    2<\2J/
FE<!ld6G6G&Gw  OOL)2<Q-/
FE<JJq:v~|DEr-   c                   | j                   ry d| _         t        | dd       \t        j                  | j                  j
                        5  | j                  j                  d d d | j                  g       d d d        y y # 1 sw Y   y xY w)NTr   )rO   rP   rQ   rR   r   r4   rS   r   r>   rT   s     r.   rS   zTFDeiTPatchEmbeddings.build   s}    ::
4t,8t334 M%%tT49J9J&KLM M 9M Ms   *A??Br3   r   r   r   )r}   r!   r   r!   r   r'   r(   r)   r*   r8   r   rS   r   r   s   @r.   r:   r:      s    
"
Mr-   r:   c                  N     e Zd Zd fdZddZ	 d	 	 	 	 	 	 	 	 	 ddZd	dZ xZS )
TFDeiTSelfAttentionc                   t        |   d
i | |j                  |j                  z  dk7  r&t	        d|j                   d|j                   d      |j                  | _        t        |j                  |j                  z        | _        | j                  | j                  z  | _        t        j                  | j                        | _
        t        j                  j                  | j                  t        |j                        d      | _        t        j                  j                  | j                  t        |j                        d      | _        t        j                  j                  | j                  t        |j                        d      | _        t        j                  j'                  |j(                  	      | _        || _        y )Nr   zThe hidden size (z6) is not a multiple of the number of attention heads ()queryunitskernel_initializerr4   keyvaluerater,   )r7   r8   rK   num_attention_headsr   ra   attention_head_sizeall_head_sizerb   rc   sqrt_att_head_sizer   r;   Denser   initializer_ranger   r   r   r<   attention_probs_dropout_probr5   r3   r>   r3   r?   r@   s      r.   r8   zTFDeiTSelfAttention.__init__   s   "6" : ::a?#F$6$6#7 8''-'A'A&B!E 
 $*#=#= #&v'9'9F<V<V'V#W !558P8PP"&))D,D,D"E\\''$$IaIa9bip ( 

 <<%%$$IaIa9bin & 
 \\''$$IaIa9bip ( 

 ||++1T1T+Ur-   c                    t        j                  ||d| j                  | j                  f      }t        j                  |g d      S )NrV   tensorrC   r   rH   r   r   r[   )rQ   r`   r   r   rf   )r>   r   r   s      r.   transpose_for_scoresz(TFDeiTSelfAttention.transpose_for_scores  s;    6*b$BZBZ\`\t\t1uv ||F66r-   c                   t        |      d   }| j                  |      }| j                  |      }| j                  |      }| j	                  ||      }	| j	                  ||      }
| j	                  ||      }t        j                  |	|
d      }t        j                  | j                  |j                        }t        j                  ||      }t        |d      }| j                  ||      }|t        j                  ||      }t        j                  ||      }t        j                  |g d	
      }t        j                  ||d| j                   f      }|r||f}|S |f}|S )Nr   inputsT)transpose_bru   rV   )r"   r^   r   ry   r   r[   r   )r   r   r   r   r   rQ   matmulr{   r   rv   divider   r5   multiplyrf   r`   r   )r>   r%   	head_maskoutput_attentionsry   r   mixed_query_layermixed_key_layermixed_value_layerquery_layer	key_layervalue_layerattention_scoresdkattention_probsattention_outputoutputss                    r.   r   zTFDeiTSelfAttention.call  se     .q1
 JJmJ<((-(8 JJmJ<//0A:N--ozJ	//0A:N 99[)NWWT,,4D4J4JK99%5r: )0@rJ ,,o,Q   kk/9EO99_kB<<(8|L ::-=jRTVZVhVhEij9J#_5 RbPcr-   c                   | j                   ry d| _         t        | dd       dt        j                  | j                  j
                        5  | j                  j                  d d | j                  j                  g       d d d        t        | dd       dt        j                  | j                  j
                        5  | j                  j                  d d | j                  j                  g       d d d        t        | dd       et        j                  | j                  j
                        5  | j                  j                  d d | j                  j                  g       d d d        y y # 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   y xY w)NTr   r   r   )rO   rP   rQ   rR   r   r4   rS   r3   rK   r   r   r   s     r.   rS   zTFDeiTSelfAttention.buildH  s9   ::
4$'3tzz/ H

  $dkk.E.E!FGH4%1txx}}- FdDKK,C,CDEF4$'3tzz/ H

  $dkk.E.E!FGH H 4H HF FH Hs$   3E*<3E6-3F*E36E?Fr3   r   )r   r!   r   ra   r   r!   r   
r%   r!   r   r!   r   r   ry   r   r   Tuple[tf.Tensor]r   )r'   r(   r)   r8   r   r   rS   r   r   s   @r.   r   r      sN    47 ' ' '  	'
 ' 
'RHr-   r   c                  6     e Zd ZdZd fdZdddZddZ xZS )	TFDeiTSelfOutputz
    The residual connection is defined in TFDeiTLayer instead of here (as is the case with other models), due to the
    layernorm applied before each block.
    c                   t        |   di | t        j                  j	                  |j
                  t        |j                        d      | _        t        j                  j                  |j                        | _        || _        y Ndenser   r   r,   r7   r8   r   r;   r   rK   r   r   r   r<   r=   r5   r3   r   s      r.   r8   zTFDeiTSelfOutput.__init__^  o    "6"\\''$$IaIa9bip ( 

 ||++1K1K+Lr-   c                P    | j                  |      }| j                  ||      }|S Nr   r   r   r5   r>   r%   input_tensorry   s       r.   r   zTFDeiTSelfOutput.callg  s*    

-
8MHMr-   c                (   | j                   ry d| _         t        | dd       et        j                  | j                  j
                        5  | j                  j                  d d | j                  j                  g       d d d        y y # 1 sw Y   y xY wNTr   	rO   rP   rQ   rR   r   r4   rS   r3   rK   r   s     r.   rS   zTFDeiTSelfOutput.buildm  }    ::
4$'3tzz/ H

  $dkk.E.E!FGH H 4H H   3BBr   r   r%   r!   r   r!   ry   r   r   r!   r   r   r   s   @r.   r   r   X  s    
Hr-   r   c                  L     e Zd Zd fdZd Z	 d	 	 	 	 	 	 	 	 	 ddZddZ xZS )	TFDeiTAttentionc                l    t        |   di | t        |d      | _        t	        |d      | _        y )N	attentionr6   outputr,   )r7   r8   r   self_attentionr   dense_outputr   s      r.   r8   zTFDeiTAttention.__init__x  s1    "6"1&{K,V(Cr-   c                    t         r   NotImplementedError)r>   headss     r.   prune_headszTFDeiTAttention.prune_heads~  s    !!r-   c                p    | j                  ||||      }| j                  |d   ||      }|f|dd  z   }|S )Nr%   r   r   ry   r   r%   r   ry   r   )r   r   )r>   r   r   r   ry   self_outputsr   r   s           r.   r   zTFDeiTAttention.call  sb     **&)O`ks + 
  ,,&q/x - 
 $%QR(88r-   c                   | j                   ry d| _         t        | dd       Mt        j                  | j                  j
                        5  | j                  j                  d        d d d        t        | dd       Nt        j                  | j                  j
                        5  | j                  j                  d        d d d        y y # 1 sw Y   exY w# 1 sw Y   y xY w)NTr   r   )rO   rP   rQ   rR   r   r4   rS   r   r   s     r.   rS   zTFDeiTAttention.build  s    ::
4)40<t22778 0##))$/04.:t00556 .!!''-. . ;0 0. .   C%CCC r   r   )
r   r!   r   r!   r   r   ry   r   r   r   r   )r'   r(   r)   r8   r   r   rS   r   r   s   @r.   r   r   w  sM    D"    	
  
"	.r-   r   c                  0     e Zd Zd fdZddZddZ xZS )TFDeiTIntermediatec                T   t        |   di | t        j                  j	                  |j
                  t        |j                        d      | _        t        |j                  t              r"t        |j                        | _        || _        y |j                  | _        || _        y )Nr   r   r,   )r7   r8   r   r;   r   intermediate_sizer   r   r   r   
hidden_actstrr	   intermediate_act_fnr3   r   s      r.   r8   zTFDeiTIntermediate.__init__  s    "6"\\''**vOgOg?hov ( 

 f''-'89J9J'KD$  (.'8'8D$r-   c                L    | j                  |      }| j                  |      }|S )Nr   )r   r   )r>   r%   s     r.   r   zTFDeiTIntermediate.call  s(    

-
800?r-   c                (   | j                   ry d| _         t        | dd       et        j                  | j                  j
                        5  | j                  j                  d d | j                  j                  g       d d d        y y # 1 sw Y   y xY wr   r   r   s     r.   rS   zTFDeiTIntermediate.build  r   r   r   r%   r!   r   r!   r   r'   r(   r)   r8   r   rS   r   r   s   @r.   r   r     s    Hr-   r   c                  2     e Zd Zd fdZdddZddZ xZS )TFDeiTOutputc                   t        |   di | t        j                  j	                  |j
                  t        |j                        d      | _        t        j                  j                  |j                        | _        || _        y r   r   r   s      r.   r8   zTFDeiTOutput.__init__  r   r-   c                Z    | j                  |      }| j                  ||      }||z   }|S r   r   r   s       r.   r   zTFDeiTOutput.call  s4    

-
8MHM%4r-   c                (   | j                   ry d| _         t        | dd       et        j                  | j                  j
                        5  | j                  j                  d d | j                  j                  g       d d d        y y # 1 sw Y   y xY wr   )	rO   rP   rQ   rR   r   r4   rS   r3   r   r   s     r.   rS   zTFDeiTOutput.build  s}    ::
4$'3tzz/ N

  $dkk.K.K!LMN N 4N Nr   r   r   r   r   r  r   s   @r.   r  r    s    Nr-   r  c                  J     e Zd ZdZd fdZ	 d	 	 	 	 	 	 	 	 	 ddZddZ xZS )	TFDeiTLayerz?This corresponds to the Block class in the timm implementation.c                ^   t        |   di | t        |d      | _        t	        |d      | _        t        |d      | _        t        j                  j                  |j                  d      | _        t        j                  j                  |j                  d      | _        || _        y )	Nr   r6   intermediater   layernorm_beforeepsilonr4   layernorm_afterr,   )r7   r8   r   r   r   r  r  deit_outputr   r;   LayerNormalizationlayer_norm_epsr  r  r3   r   s      r.   r8   zTFDeiTLayer.__init__  s    "6"(kB.vNK'X> % ? ?H]H]dv ? w$||>>vG\G\ct>ur-   c                    | j                  | j                  ||      |||      }|d   }||z   }| j                  ||      }| j                  ||      }| j	                  |||      }|f|dd  z   }	|	S )Nr   )r   r   r   ry   r   )r%   ry   r   r   )r   r  r  r  r  )
r>   r%   r   r   ry   attention_outputsr   layer_outputintermediate_outputr   s
             r.   r   zTFDeiTLayer.call  s     !NN..mh.W/ + 
 -Q/ )=8 ++=8+T"//lU]/^ ''-MT\ ( 
  /$5ab$99r-   c                   | j                   ry d| _         t        | dd       Mt        j                  | j                  j
                        5  | j                  j                  d        d d d        t        | dd       Mt        j                  | j                  j
                        5  | j                  j                  d        d d d        t        | dd       Mt        j                  | j                  j
                        5  | j                  j                  d        d d d        t        | dd       dt        j                  | j                  j
                        5  | j                  j                  d d | j                  j                  g       d d d        t        | dd       et        j                  | j                  j
                        5  | j                  j                  d d | j                  j                  g       d d d        y y # 1 sw Y   xY w# 1 sw Y   UxY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   y xY w)NTr   r  r  r  r  )rO   rP   rQ   rR   r   r4   rS   r  r  r  r3   rK   r  r   s     r.   rS   zTFDeiTLayer.build  s   ::
4d+7t~~223 +$$T*+4.:t00556 .!!''-.4-9t//445 -  &&t,-4+T2>t4499: S%%++T49P9P,QRS4*D1=t33889 R$$**D$8O8O+PQR R >+ +. .- -S SR Rs<   H%H?H!3H.
3H:HH!H+.H7:Ir   r   r   r   r   r   s   @r.   r  r    sL    I	      	
  
@Rr-   r  c                  N     e Zd Zd fdZ	 d	 	 	 	 	 	 	 	 	 	 	 	 	 ddZddZ xZS )TFDeiTEncoderc                    t        |   di | t        |j                        D cg c]  }t	        |d|        c}| _        y c c}w )Nzlayer_._r6   r,   )r7   r8   rangenum_hidden_layersr  layer)r>   r3   r?   ir@   s       r.   r8   zTFDeiTEncoder.__init__  s@    "6"HMfNfNfHgh1k&!~>h
hs   Ac                    |rdnd }|rdnd }t        | j                        D ]-  \  }	}
|r||fz   } |
|||	   ||      }|d   }|s%||d   fz   }/ |r||fz   }|st        d |||fD              S t        |||      S )Nr,   r   r   r   c              3  &   K   | ]	  }||  y wr   r,   ).0vs     r.   	<genexpr>z%TFDeiTEncoder.call.<locals>.<genexpr>A  s     hqZ[Zghs   )last_hidden_stater%   r&   )	enumerater  tupler
   )r>   r%   r   r   output_hidden_statesreturn_dictry   all_hidden_statesall_attentionsr   layer_modulelayer_outputss               r.   r   zTFDeiTEncoder.call!  s     #7BD0d(4 	FOA|#$58H$H!(+#A,"3!	M *!,M !/=3C2E!E	F    1]4D Dh]4E~$Vhhh +;LYg
 	
r-   c                    | j                   ry d| _         t        | dd       K| j                  D ];  }t        j                  |j
                        5  |j                  d        d d d        = y y # 1 sw Y   IxY w)NTr  )rO   rP   r  rQ   rR   r4   rS   )r>   rT   r  s      r.   rS   zTFDeiTEncoder.buildG  sp    ::
4$'3 &]]5::. &KK%& && 4& &s   A..A7	r   r   )r%   r!   r   r!   r   r   r)  r   r*  r   ry   r   r   z*Union[TFBaseModelOutput, Tuple[tf.Tensor]]r   r  r   s   @r.   r  r    s]    i $
 $
 $
  	$

 #$
 $
 $
 
4$
L&r-   r  c                       e Zd ZeZ	 d	 	 	 	 	 	 	 d fdZd	dZd Zd Ze		 	 	 	 	 	 	 	 d
	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dd       Z
ddZ xZS )TFDeiTMainLayerc                   t        |   di | || _        t        ||d      | _        t        |d      | _        t        j                  j                  |j                  d      | _        |rt        |d      | _        y d | _        y )	Nri   )r9   r4   encoderr6   	layernormr  poolerr,   )r7   r8   r3   r0   ri   r  r3  r   r;   r  r  r4  TFDeiTPoolerr5  r>   r3   add_pooling_layerr9   r?   r@   s        r.   r8   zTFDeiTMainLayer.__init__U  st     	"6"*6.Wcd$V)<88AVAV]h8i=Nl69TXr-   c                .    | j                   j                  S r   )ri   r2   )r>   s    r.   get_input_embeddingsz$TFDeiTMainLayer.get_input_embeddingsa  s    ///r-   c                    t         )z
        Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base
        class PreTrainedModel
        r   )r>   heads_to_prunes     r.   _prune_headszTFDeiTMainLayer._prune_headsd  s
    
 "!r-   c                J    |t         d g| j                  j                  z  }|S r   )r   r3   r  )r>   r   s     r.   get_head_maskzTFDeiTMainLayer.get_head_maskk  s*     %%!>!>>Ir-   c	                <   ||n| j                   j                  }||n| j                   j                  }||n| j                   j                  }|t	        d      t        j                  |d      }| j                  |      }| j                  ||||      }	| j                  |	|||||      }
|
d   }| j                  ||      }| j                  | j                  ||      nd }|s|||fn|f}||
dd  z   S t        |||
j                  |
j                        S )	Nz You have to specify pixel_valuesrZ   )r~   ry   rs   )r   r   r)  r*  ry   r   rx   r   )r&  pooler_outputr%   r&   )r3   r   r)  use_return_dictr   rQ   rf   r?  ri   r3  r4  r5  r   r%   r&   )r>   r}   r~   r   r   r)  r*  rs   ry   embedding_outputencoder_outputssequence_outputpooled_outputhead_outputss                 r.   r   zTFDeiTMainLayer.calls  sZ    2C1N-TXT_T_TqTq$8$D $++JjJj 	 &1%<k$++B]B]?@@ ||L,? &&y1	??+%=	 + 
 ,,/!5# ' 
 *!,..8.LKO;;KbOhGhl?L?XO];_n^pL/!""555+-')77&11	
 	
r-   c                   | j                   ry d| _         t        | dd       Mt        j                  | j                  j
                        5  | j                  j                  d        d d d        t        | dd       Mt        j                  | j                  j
                        5  | j                  j                  d        d d d        t        | dd       dt        j                  | j                  j
                        5  | j                  j                  d d | j                  j                  g       d d d        t        | dd       Nt        j                  | j                  j
                        5  | j                  j                  d        d d d        y y # 1 sw Y   1xY w# 1 sw Y   xY w# 1 sw Y   ~xY w# 1 sw Y   y xY w)NTri   r3  r4  r5  )rO   rP   rQ   rR   ri   r4   rS   r3  r4  r3   rK   r5  r   s     r.   rS   zTFDeiTMainLayer.build  sb   ::
4t,8t334 ,%%d+,4D)5t||001 )""4()4d+7t~~223 L$$dD$++2I2I%JKL44(4t{{//0 (!!$'( ( 5, ,) )L L( (s0   F%F#?3F/0F;F #F,/F8;GTFr3   r   r8  r   r9   r   r   r   )r   r:   NNNNNNFF)r}   r   r~   r   r   r   r   Optional[bool]r)  rL  r*  rL  rs   r   ry   r   r   z:Union[TFBaseModelOutputWithPooling, Tuple[tf.Tensor, ...]]r   )r'   r(   r)   r   config_classr8   r:  r=  r?  r   r   rS   r   r   s   @r.   r1  r1  Q  s    L Z_
Y 
Y59
YRV
Y	
Y0"  *.,0&*,0/3&*).;
&;
 *;
 $	;

 *;
 -;
 $;
 #';
 ;
 
D;
 ;
z(r-   r1  c                      e Zd ZdZeZdZdZy)TFDeiTPreTrainedModelz
    An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
    models.
    deitr}   N)r'   r(   r)   r*   r   rM  base_model_prefixmain_input_namer,   r-   r.   rO  rO    s    
 L$Or-   rO  aR  
    This model is a TensorFlow
    [keras.layers.Layer](https://www.tensorflow.org/api_docs/python/tf/keras/layers/Layer). Use it as a regular
    TensorFlow Module and refer to the TensorFlow documentation for all matter related to general usage and behavior.

    Parameters:
        config ([`DeiTConfig`]): Model configuration class with all the parameters of the model.
            Initializing with a config file does not load the weights associated with the model, only the
            configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
a  
    Args:
        pixel_values (`tf.Tensor` of shape `(batch_size, num_channels, height, width)`):
            Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See
            [`DeiTImageProcessor.__call__`] for details.

        head_mask (`tf.Tensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*):
            Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`:

            - 1 indicates the head is **not masked**,
            - 0 indicates the head is **masked**.

        output_attentions (`bool`, *optional*):
            Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
            tensors for more detail.
        output_hidden_states (`bool`, *optional*):
            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
            more detail.
        interpolate_pos_encoding (`bool`, *optional*, defaults to `False`):
            Whether to interpolate the pre-trained position encodings.
        return_dict (`bool`, *optional*):
            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
z^The bare DeiT Model transformer outputting raw hidden-states without any specific head on top.c            	           e Zd Z	 d	 	 	 	 	 	 	 d fdZe ee       eee	e
de      	 	 	 	 	 	 	 	 d	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d	d                     Zd
dZ xZS )TFDeiTModelc                N    t        |   |fi | t        |||d      | _        y )NrP  r8  r9   r4   )r7   r8   r1  rP  r7  s        r.   r8   zTFDeiTModel.__init__  s.     	*6*#&7]c
	r-   vision)
checkpointoutput_typerM  modalityexpected_outputc	           
     8    | j                  ||||||||      }	|	S )N)r}   r~   r   r   r)  r*  rs   ry   )rP  )
r>   r}   r~   r   r   r)  r*  rs   ry   r   s
             r.   r   zTFDeiTModel.call  s6    ( ))%+/!5#%=  	
 r-   c                    | j                   ry d| _         t        | dd       Nt        j                  | j                  j
                        5  | j                  j                  d        d d d        y y # 1 sw Y   y xY w)NTrP  )rO   rP   rQ   rR   rP  r4   rS   r   s     r.   rS   zTFDeiTModel.build!  se    ::
4&2tyy~~. &		%& & 3& &s   A11A:rI  rJ  rK  )r}   r   r~   r   r   r   r   rL  r)  rL  r*  rL  rs   r   ry   r   r   z*Union[Tuple, TFBaseModelOutputWithPooling]r   )r'   r(   r)   r8   r   r   DEIT_INPUTS_DOCSTRINGr   _CHECKPOINT_FOR_DOCr   _CONFIG_FOR_DOC_EXPECTED_OUTPUT_SHAPEr   rS   r   r   s   @r.   rT  rT    s     Z_
 
59
RV
	
 *+@A&0$. *.,0&*,0/3&*).& * $	
 * - $ #'  
4 B .&r-   rT  c                  0     e Zd Zd fdZddZddZ xZS )r6  c                    t        |   di | t        j                  j	                  |j
                  t        |j                        dd      | _        || _	        y )Ntanhr   )r   r   
activationr4   r,   )
r7   r8   r   r;   r   rK   r   r   r   r3   r   s      r.   r8   zTFDeiTPooler.__init__,  sT    "6"\\''$$.v/G/GH	 ( 

 r-   c                <    |d d df   }| j                  |      }|S )Nr   r   )r   )r>   r%   first_token_tensorrF  s       r.   r   zTFDeiTPooler.call7  s*     +1a40

*<
=r-   c                (   | j                   ry d| _         t        | dd       et        j                  | j                  j
                        5  | j                  j                  d d | j                  j                  g       d d d        y y # 1 sw Y   y xY wr   r   r   s     r.   rS   zTFDeiTPooler.build?  r   r   r   r  r   r  r   s   @r.   r6  r6  +  s    	Hr-   r6  c                  ,     e Zd ZdZd fdZddZ xZS )TFDeitPixelShufflez0TF layer implementation of torch.nn.PixelShufflec                x    t        |   di | t        |t              r|dk  rt	        d|       || _        y )NrH   z1upscale_factor must be an integer value >= 2 got r,   )r7   r8   r   ra   r   upscale_factor)r>   rl  r?   r@   s      r.   r8   zTFDeitPixelShuffle.__init__K  sA    "6".#..12DPQ_P`abb,r-   c           
        |}t        |      \  }}}}| j                  dz  }t        ||z        }t        j                  t        |      D 	cg c]  }t        |      D ]
  }	||	|z  z     c}	}g      }
t        j                  |t        j                  |
|dg      d      }t        j                  j                  || j                  d      }|S c c}	}w )NrH   r   rV   )paramsindices
batch_dimsNHWC)
block_sizedata_format)
r   rl  ra   rQ   constantr  gatherrz   nndepth_to_space)r>   r   r%   r   r   num_input_channelsblock_size_squaredoutput_depthr   jpermutations              r.   r   zTFDeitPixelShuffle.callQ  s    /9-/H,
Aq,!00!3-0BBC
 kk278J2KiQUZ[gUhiPQa!(((i(ij
 		V`bcUd@ertu,,]tGZGZhn,o	 js   C
)rl  ra   r   r   )r   r!   r   r!   )r'   r(   r)   r*   r8   r   r   r   s   @r.   rj  rj  H  s    :-r-   rj  c                  2     e Zd Zd fdZdddZddZ xZS )TFDeitDecoderc                    t        |   di | t        j                  j	                  |j
                  dz  |j                  z  dd      | _        t        |j
                  d      | _	        || _
        y )NrH   r   0)filtersr   r4   1r6   r,   )r7   r8   r   r;   r   encoder_strider   conv2drj  pixel_shuffler3   r   s      r.   r8   zTFDeitDecoder.__init__c  sk    "6"ll))))1,v/B/BBPQX[ * 
 00E0ECPr-   c                N    |}| j                  |      }| j                  |      }|S r   )r  r  )r>   r   ry   r%   s       r.   r   zTFDeitDecoder.callk  s+    M2**=9r-   c                   | j                   ry d| _         t        | dd       et        j                  | j                  j
                        5  | j                  j                  d d d | j                  j                  g       d d d        t        | dd       Nt        j                  | j                  j
                        5  | j                  j                  d        d d d        y y # 1 sw Y   exY w# 1 sw Y   y xY w)NTr  r  )
rO   rP   rQ   rR   r  r4   rS   r3   rK   r  r   s     r.   rS   zTFDeitDecoder.buildq  s    ::
44(4t{{//0 O!!4tT[[5L5L"MNO4$/;t11667 /""((./ / <O O/ /s   4C#=C/#C,/C8r   r   )r   r!   ry   r   r   r!   r   r  r   s   @r.   r~  r~  b  s    	/r-   r~  zvDeiT Model with a decoder on top for masked image modeling, as proposed in [SimMIM](https://arxiv.org/abs/2111.09886).c                       e Zd Zd fdZe ee       eee	      	 	 	 	 	 	 	 	 d	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dd                     Z
ddZ xZS )	TFDeiTForMaskedImageModelingc                p    t         |   |       t        |ddd      | _        t	        |d      | _        y )NFTrP  rV  decoderr6   )r7   r8   r1  rP  r~  r  r>   r3   r@   s     r.   r8   z%TFDeiTForMaskedImageModeling.__init__  s2     #FeTX_ef	$V)<r-   rY  rM  c	           
        ||n| j                   j                  }| j                  ||||||||      }	|	d   }
|
ddddf   }
t        |
      \  }}}t	        |dz        x}}t        j                  |
||||f      }
| j                  |
|      }t        j                  |d      }d}|| j                   j                  | j                   j                  z  }t        j                  |d||f      }t        j                  || j                   j                  d      }t        j                  || j                   j                  d	      }t        j                  |d      }t        j                  |t
        j                        }t        j                   j#                  t        j                  |d
      t        j                  |d
            }t        j                  |d      }t        j$                  ||z        }t        j$                  |      dz   | j                   j&                  z  }||z  }t        j                  |d      }|s|f|	dd z   }||f|z   S |S t)        |||	j*                  |	j,                        S )a  
        bool_masked_pos (`tf.Tensor` of type bool and shape `(batch_size, num_patches)`):
            Boolean masked positions. Indicates which patches are masked (1) and which aren't (0).

        Returns:

        Examples:
        ```python
        >>> from transformers import AutoImageProcessor, TFDeiTForMaskedImageModeling
        >>> import tensorflow as tf
        >>> from PIL import Image
        >>> import requests

        >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
        >>> image = Image.open(requests.get(url, stream=True).raw)

        >>> image_processor = AutoImageProcessor.from_pretrained("facebook/deit-base-distilled-patch16-224")
        >>> model = TFDeiTForMaskedImageModeling.from_pretrained("facebook/deit-base-distilled-patch16-224")

        >>> num_patches = (model.config.image_size // model.config.patch_size) ** 2
        >>> pixel_values = image_processor(images=image, return_tensors="tf").pixel_values
        >>> # create random boolean mask of shape (batch_size, num_patches)
        >>> bool_masked_pos = tf.cast(tf.random.uniform((1, num_patches), minval=0, maxval=2, dtype=tf.int32), tf.bool)

        >>> outputs = model(pixel_values, bool_masked_pos=bool_masked_pos)
        >>> loss, reconstructed_pixel_values = outputs.loss, outputs.reconstruction
        >>> list(reconstructed_pixel_values.shape)
        [1, 3, 224, 224]
        ```N)r~   r   r   r)  r*  rs   ry   r   r   rV   g      ?rx   )r   r   r   rH   rH   )r   rH   r   r   gh㈵>)r   )lossreconstructionr%   r&   )r3   rB  rP  r   ra   rQ   r`   r  rf   r   r_   r|   rh   r{   float32r   lossesmean_absolute_error
reduce_sumr   r   r%   r&   )r>   r}   r~   r   r   r)  r*  rs   ry   r   rE  r   sequence_lengthr   rj   rk   reconstructed_pixel_valuesmasked_im_lossrX   r   reconstruction_loss
total_lossnum_masked_pixelsr   s                           r.   r   z!TFDeiTForMaskedImageModeling.call  sU   V &1%<k$++B]B]))+/!5#%=  	
 "!* *!QrT'24>4O1
O\_c122**_z65R^6_` &*\\/H\%U" &(\\2Ll%["&;;))T[[-C-CCD jj2tT:JKO99_dkk.D.DaHD99T4;;#9#91=D>>$*D774,D"',,"B"B\<87F#
 #%..1Da"H':T'ABJ!#t!4t!;t{{?W?W W'*;;NZZ=N02WQR[@F3A3M^%.YSYY*5!//))	
 	
r-   c                   | j                   ry d| _         t        | dd       Mt        j                  | j                  j
                        5  | j                  j                  d        d d d        t        | dd       Nt        j                  | j                  j
                        5  | j                  j                  d        d d d        y y # 1 sw Y   exY w# 1 sw Y   y xY w)NTrP  r  )rO   rP   rQ   rR   rP  r4   rS   r  r   s     r.   rS   z"TFDeiTForMaskedImageModeling.build  s    ::
4&2tyy~~. &		%&4D)5t||001 )""4() ) 6& &) )r   r   rK  )r}   r   r~   r   r   r   r   rL  r)  rL  r*  rL  rs   r   ry   r   r   z)Union[tuple, TFMaskedImageModelingOutput]r   )r'   r(   r)   r8   r   r   r^  r   r   r`  r   rS   r   r   s   @r.   r  r  }  s    = *+@A+FUde *.,0&*,0/3&*).a
&a
 *a
 $	a

 *a
 -a
 $a
 #'a
 a
 
3a
 f B a
F	)r-   r  z
    DeiT Model transformer with an image classification head on top (a linear layer on top of the final hidden state of
    the [CLS] token) e.g. for ImageNet.
    c                       e Zd Zd fdZe ee       eee	      	 	 	 	 	 	 	 	 d	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dd                     Z
ddZ xZS )	TFDeiTForImageClassificationc                :   t         |   |       |j                  | _        t        |dd      | _        |j                  dkD  r+t
        j                  j                  |j                  d      n t
        j                  j                  dd      | _	        || _
        y )NFrP  r8  r4   r   
classifierr6   linear)r7   r8   
num_labelsr1  rP  r   r;   r   
Activationr  r3   r  s     r.   r8   z%TFDeiTForImageClassification.__init__  s      ++#Fe&Q	
   1$ LLv00|D(((E 	
 r-   r  c	           	     B   ||n| j                   j                  }| j                  |||||||      }	|	d   }
| j                  |
dddddf         }|dn| j	                  ||      }|s|f|	dd z   }||f|z   S |S t        |||	j                  |	j                        S )a  
        labels (`tf.Tensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the image classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).

        Returns:

        Examples:

        ```python
        >>> from transformers import AutoImageProcessor, TFDeiTForImageClassification
        >>> import tensorflow as tf
        >>> from PIL import Image
        >>> import requests

        >>> keras.utils.set_random_seed(3)  # doctest: +IGNORE_RESULT
        >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
        >>> image = Image.open(requests.get(url, stream=True).raw)

        >>> # note: we are loading a TFDeiTForImageClassificationWithTeacher from the hub here,
        >>> # so the head will be randomly initialized, hence the predictions will be random
        >>> image_processor = AutoImageProcessor.from_pretrained("facebook/deit-base-distilled-patch16-224")
        >>> model = TFDeiTForImageClassification.from_pretrained("facebook/deit-base-distilled-patch16-224")

        >>> inputs = image_processor(images=image, return_tensors="tf")
        >>> outputs = model(**inputs)
        >>> logits = outputs.logits
        >>> # model predicts one of the 1000 ImageNet classes
        >>> predicted_class_idx = tf.math.argmax(logits, axis=-1)[0]
        >>> print("Predicted class:", model.config.id2label[int(predicted_class_idx)])
        Predicted class: little blue heron, Egretta caerulea
        ```Nr   r   r)  r*  rs   ry   r   r   )r  r"   r%   r&   )r3   rB  rP  r  hf_compute_lossr   r%   r&   )r>   r}   r   labelsr   r)  r*  rs   ry   r   rE  r"   r  r   s                 r.   r   z!TFDeiTForImageClassification.call  s    ^ &1%<k$++B]B]))/!5#%=  
 "!*Aq!9: ~t4+?+?+OY,F)-)9TGf$EvE&!//))	
 	
r-   c                   | j                   ry d| _         t        | dd       Mt        j                  | j                  j
                        5  | j                  j                  d        d d d        t        | dd       et        j                  | j                  j
                        5  | j                  j                  d d | j                  j                  g       d d d        y y # 1 sw Y   |xY w# 1 sw Y   y xY w)NTrP  r  )
rO   rP   rQ   rR   rP  r4   rS   r  r3   rK   r   s     r.   rS   z"TFDeiTForImageClassification.build^  s    ::
4&2tyy~~. &		%&4t,8t334 M%%tT4;;3J3J&KLM M 9& &M Ms   C"%3C."C+.C7r   rK  )r}   r   r   r   r  r   r   rL  r)  rL  r*  rL  rs   r   ry   r   r   z)Union[tf.Tensor, TFImageClassifierOutput]r   )r'   r(   r)   r8   r   r   r^  r   r   r`  r   rS   r   r   s   @r.   r  r    s     *+@A+BQ`a *.&*#',0/3&*).H
&H
 $H
 !	H

 *H
 -H
 $H
 #'H
 H
 
3H
 b B H
T	Mr-   r  a  
    DeiT Model transformer with image classification heads on top (a linear layer on top of the final hidden state of
    the [CLS] token and a linear layer on top of the final hidden state of the distillation token) e.g. for ImageNet.

    .. warning::

            This model supports inference-only. Fine-tuning with distillation (i.e. with a teacher) is not yet
            supported.
    c                       e Zd Zd fdZe ee       eee	e
e      	 	 	 	 	 	 	 d	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dd                     ZddZ xZS )	'TFDeiTForImageClassificationWithTeacherc                   t         |   |       |j                  | _        t        |dd      | _        |j                  dkD  r+t
        j                  j                  |j                  d      n t
        j                  j                  dd      | _	        |j                  dkD  r+t
        j                  j                  |j                  d      n t
        j                  j                  dd      | _
        || _        y )	NFrP  r  r   cls_classifierr6   r  distillation_classifier)r7   r8   r  r1  rP  r   r;   r   r  r  r  r3   r  s     r.   r8   z0TFDeiTForImageClassificationWithTeacher.__init__w  s      ++#Fe&Q	
   1$ LLv007GH((8H(I 	   1$ LLv007PQ((8Q(R 	$
 r-   )rX  rY  rM  r[  c           	     R   ||n| j                   j                  }| j                  |||||||      }|d   }	| j                  |	d d dd d f         }
| j	                  |	d d dd d f         }|
|z   dz  }|s||
|f|dd  z   }|S t        ||
||j                  |j                        S )Nr  r   r   rH   )r"   r#   r$   r%   r&   )r3   rB  rP  r  r  r    r%   r&   )r>   r}   r   r   r)  r*  rs   ry   r   rE  r#   r$   r"   r   s                 r.   r   z,TFDeiTForImageClassificationWithTeacher.call  s    $ &1%<k$++B]B]))/!5#%=  
 "!*((Aq)AB
"::?1aQR7;ST 22a7j*=>LFM<! 3!//))
 	
r-   c                   | j                   ry d| _         t        | dd       Mt        j                  | j                  j
                        5  | j                  j                  d        d d d        t        | dd       dt        j                  | j                  j
                        5  | j                  j                  d d | j                  j                  g       d d d        t        | dd       et        j                  | j                  j
                        5  | j                  j                  d d | j                  j                  g       d d d        y y # 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   y xY w)NTrP  r  r  )rO   rP   rQ   rR   rP  r4   rS   r  r3   rK   r  r   s     r.   rS   z-TFDeiTForImageClassificationWithTeacher.build  s3   ::
4&2tyy~~. &		%&4)40<t22778 Q##))4t{{7N7N*OPQ42D9Et;;@@A Z,,22D$@W@W3XYZ Z F& &Q QZ Zs$   E%3E3E+EE(+E4r   )NNNNNFF)r}   r   r   r   r   rL  r)  rL  r*  rL  rs   r   ry   r   r   z;Union[tuple, TFDeiTForImageClassificationWithTeacherOutput]r   )r'   r(   r)   r8   r   r   r^  r   _IMAGE_CLASS_CHECKPOINTr    r`  _IMAGE_CLASS_EXPECTED_OUTPUTr   rS   r   r   s   @r.   r  r  j  s    & *+@A*A$4	 *.&*,0/3&*).(
&(
 $(
 *	(

 -(
 $(
 #'(
 (
 
E(
 B (
TZr-   r  )Gr*   
__future__r   collections.abcr   rb   dataclassesr   typingr   r   r   
tensorflowrQ   activations_tfr	   modeling_tf_outputsr
   r   r   r   modeling_tf_utilsr   r   r   r   r   r   tf_utilsr   r   utilsr   r   r   r   r   r   configuration_deitr   
get_loggerr'   loggerr`  r_  ra  r  r  r    r;   Layerr0   r:   r   r   r   r   r  r  r  r1  rO  DEIT_START_DOCSTRINGr^  rT  r6  rj  r~  r  r  r  r,   r-   r.   <module>r     s    "   ! ) )  /   3  + 
		H	%  A &  E 1  /K / /<ju||)) jZ*MELL.. *M\WH%,,,, WHvHu||)) H>$.ell(( $.PH++ H<N5<<%% N4@R%,,$$ @RH3&ELL&& 3&l n(ell(( n( n(d%- %	  2 d0&' 0&	0&hH5<<%% H:++ 4/ELL&& /6 3
v)#8 v)
v)r  eM#8:V eMeMP  RZ.C RZRZr-   