sg-dZddlmZddlZddlZddlZddlZddlmZm Z m Z ddl Z ddl ZddlmZddlmZddlmZmZmZmZdd lmZmZmZmZmZmZmZdd l m!Z!m"Z"m#Z#dd l$m%Z%m&Z&m'Z'm(Z(d d l)m*Z*e'jVe,Z-dZ.Gddej^j`Z1Gddej^j`Z2Gddej^j`Z3Gddej^j`Z4Gddej^j`Z5Gddej^j`Z6Gddej^j`Z7Gddej^j`Z8eGdd ej^j`Z9Gd!d"eZ:d#Z;d$Ze%d'e;Gd(d)e:Z?e%d*e;Gd+d,e:eZ@e%d-e;Gd.d/e:ZAy)0zTF 2.0 T5 model.) annotationsN)OptionalTupleUnion) dynamic_slice)get_tf_activation)TFBaseModelOutput+TFBaseModelOutputWithPastAndCrossAttentionsTFSeq2SeqLMOutputTFSeq2SeqModelOutput)TFCausalLanguageModelingLossTFModelInputTypeTFPreTrainedModelget_initializerkeraskeras_serializable unpack_inputs)check_embeddings_within_bounds shape_liststable_softmax)add_start_docstrings%add_start_docstrings_to_model_forwardloggingreplace_return_docstrings)T5Configrc0eZdZdfd ZfdZdZxZS) TFT5LayerNormc @t|di|||_||_y)zb Construct a layernorm module in the T5 style No bias and no subtraction of mean. N)super__init__variance_epsilon hidden_size)selfr%epsilonkwargs __class__s X/var/www/html/venv/lib/python3.12/site-packages/transformers/models/t5/modeling_tf_t5.pyr#zTFT5LayerNorm.__init__Es% "6" '&cl|jd|jfd|_t||y)z!Build shared word embedding layerweightones)shape initializerN) add_weightr%r-r"build)r& input_shaper)s r*r2zTFT5LayerNorm.buildMs/ooht7G7G6IW]o^   k"r+ctjjtjj|dd}|tjj ||j zz}|j |zS)NT)axiskeepdims)tfmath reduce_meansquarersqrtr$r-)r& hidden_statesvariances r*callzTFT5LayerNorm.callRs\77&&rww~~m'D2X\&]% hAVAV6V(WW {{]**r+)gư>)__name__ __module__ __qualname__r#r2r? __classcell__r)s@r*rrDs'# +r+rc.eZdZfdZddZddZxZS)TFT5DenseActDensec vt|di|tjj d|j |j dzz}tjj d|j |jdzz}tjj|jdd||_ tjj|j dd||_ tjj|j|_t|j |_||_y) Nr࿩meanstddevFwiuse_biasnamekernel_initializerwor!)r"r#r initializers RandomNormalinitializer_factord_modeld_fflayersDenserLrQDropout dropout_ratedropoutr dense_act_fnactconfigr&r^r(wi_initializerwo_initializerr)s r*r#zTFT5DenseActDense.__init__Ys "6"++886448LM9 ++88644 T8IJ9 ,,$$ KK%d~% ,,$$ NNU.% ||++F,?,?@ $V%8%89 r+c|j|}|j|}|j||}|j|}|SNtraining)rLr]r[rQ)r&r=res r*r?zTFT5DenseActDense.callksD . /  ]X F  . r+c"|jryd|_t|dddtj|jj 5|jj dd|jjgdddt|ddetj|jj 5|jj dd|jjgdddyy#1swY|xYw#1swYyxYw)NTrLrQ) builtgetattrr8 name_scoperLrOr2r^rUrQrVr&r3s r*r2zTFT5DenseActDense.buildrs ::  4t $ 0tww||, A tT4;;+>+>?@ A 4t $ 0tww||, > tT4;;+;+;<= > > 1 A A > >s 3C9<3D9DDFNr@rArBr#r?r2rCrDs@r*rFrFXs$ >r+rFc.eZdZfdZddZddZxZS)TFT5DenseGatedActDensec t|d i|tjj d|j |j dzz}tjj d|j |jdzz}tjj|jdd||_ tjj|jdd||_ tjj|j dd||_ tjj|j|_t!|j"|_||_y) NrrHrIFwi_0rMwi_1rQr!)r"r#rrRrSrTrUrVrWrXrqrrrQrYrZr[r r\r]r^r_s r*r#zTFT5DenseGatedActDense.__init__s2 "6"++886448LM9 ++88644 T8IJ9 LL&& KK%f'  LL&& KK%f'  ,,$$ NNU.% ||++F,?,?@ $V%8%89 r+c|j|j|}|j|}||z}|j||}|j |}|Src)r]rqrrr[rQ)r&r=re hidden_gelu hidden_linears r*r?zTFT5DenseGatedActDense.callsWhhtyy78  -0 #m3  ]X F  . r+c|jryd|_t|dddtj|jj 5|jj dd|jjgdddt|dddtj|jj 5|jj dd|jjgdddt|ddetj|jj 5|jj dd|jjgdddyy#1swYxYw#1swYxYw#1swYyxYw)NTrqrrrQ) rgrhr8rirqrOr2r^rUrrrQrVrjs r*r2zTFT5DenseGatedActDense.builds0 ::  4 & 2tyy~~. C tT[[-@-@ AB C 4 & 2tyy~~. C tT[[-@-@ AB C 4t $ 0tww||, > tT4;;+;+;<= > > 1  C C C C > >s$ 3E*<3E6-3F*E36E?F rkrlrmrDs@r*roro~s* >r+roc.eZdZfdZddZddZxZS) TFT5LayerFFc 0t|di||jrt|d|_nt |d|_t |j|jd|_ tjj|j|_y)NDenseReluDenserO layer_normr'rOr!)r"r# is_gated_actrorzrFrrUlayer_norm_epsilonr|rrWrYrZr[r&r^r(r)s r*r#zTFT5LayerFF.__init__ss "6"   "8FV"WD "3FAQ"RD '@Y@Y`lm||++F,?,?@ r+cz|j|}|j||}||j||z}|Src)r|rzr[)r&r=renormed_hidden_states dense_outputs r*r?zTFT5LayerFF.callsD#}=**+?(*S % \H (UU r+c|jryd|_t|ddMtj|jj 5|jj ddddt|ddNtj|jj 5|jj ddddyy#1swYexYw#1swYyxYw)NTr|rz)rgrhr8rir|rOr2rzrjs r*r2zTFT5LayerFF.builds ::  4t , 8t334 ,%%d+ , 4)4 0 <t22778 0##))$/ 0 0 = , , 0 0 C %C CC rkrlrmrDs@r*rxrxsA 0r+rxceZdZejZdfd ZddZdZe d dZ dZ d dZ xZ S) TFT5Attentionc 6t|d i|ttj|_|j |_|j|_||_|j|_ |j|_ |j|_ |j|_ |j|_|j|_|j |jz|_t$j&j)d|j*|j"|jzdzz}t$j&j)d|j*|j"dzz}t$j&j)d|j*|j"dzz}t$j&j)d|j*|j"dzz}t$j&j)d|j*|j"dzz|_t$j.j1|j"dd||_t$j.j1|j"dd||_t$j.j1|j"dd||_t$j.j1|jdd ||_t$j.j;|j<|_tA|_!y) NrrHrIFqrMkvor!)"r"r#nextrNEW_IDlayer_id is_decoder use_cachehas_relative_attention_biasoutput_attentionsrelative_attention_num_bucketsrelative_attention_max_distancerUd_kvkey_value_proj_dim num_headsn_heads inner_dimrrRrSrT#relative_attention_bias_initializerrWrXrrrrrYrZr[set pruned_heads) r&r^rr( q_initializer k_initializer v_initializer o_initializerr)s r*r#zTFT5Attention.__init__s "6"]112  ++))+F(!'!9!9.4.S.S+/5/U/U,~~ "(++'' (?(??**77644$JaJa9afj8jk8 **776448LM8 **776448LM8 **776448LM8 493E3E3R3R6448LM4S4 0## NNU$ ## NNU$ ## NNU$ ## LL5s}$ ||++F,?,?@ Er+c|jryd|_|jrWtjd5|j d|j |j g|j|_dddt|ddZtj|jj5|jjdd|jgdddt|ddZtj|jj5|jjdd|jgdddt|ddZtj|jj5|jjdd|jgdddt|dd[tj|j j5|j jdd|j"gdddyy#1swYxYw#1swYNxYw#1swYxYw#1swYxYw#1swYyxYw) NTrelative_attention_bias embeddings)rOr/r0rrrr)rgrr8rir1rrrrrhrrOr2rUrrrrrjs r*r2zTFT5Attention.builds ::   + +89 /3%>> M $ H H0?0,  4d # /tvv{{+ 9 dD$,,78 9 4d # /tvv{{+ 9 dD$,,78 9 4d # /tvv{{+ 9 dD$,,78 9 4d # /tvv{{+ ; dD$..9: ; ; 0   9 9 9 9 9 9 ; ;s;:H.)H#)H0<)H<#)IH #H-0H9<IIctrlNotImplementedError)r&headss r* prune_headszTFT5Attention.prune_heads!!r+c 6d}|rj|dz}|tjtjj|d|j|zz }tjj |}n!tjj |d }|dz}tjj||}|tjtjjtj|tjtj|tjz tj||z z ||z z|jz}tjj ||dz }|tj|||z }|S)a Adapted from Mesh Tensorflow: https://github.com/tensorflow/mesh/blob/0cb87fe07da627bf0b7e60475d59f95ed6b5be3d/mesh_tensorflow/transformer/transformer_layers.py#L593 Translate relative position to a bucket number for relative attention. The relative position is defined as memory_position - query_position, i.e. the distance in tokens from the attending position to the attended-to position. If bidirectional=False, then positive relative positions are invalid. We use smaller buckets for small absolute relative_position and larger buckets for larger absolute relative_positions. All relative positions >=max_distance map to the same bucket. All relative positions <=-max_distance map to the same bucket. This should allow for more graceful generalization to longer sequences than the model has been trained on Args: relative_position: an int32 Tensor bidirectional: a boolean - whether the attention is bidirectional num_buckets: an integer max_distance: an integer Returns: a Tensor with the same shape as relative_position, containing int32 values in the range [0, num_buckets) rdtyper) r8castr9greaterrabsminimumlesslogfloat32where)relative_position bidirectional num_buckets max_distancerelative_buckets max_exactis_smallrelative_position_if_larges r*_relative_position_bucketz'TFT5Attention._relative_position_bucketsU,  A K (91=EVE\E\]`kk  !# ,= > !#1BA!F F 1$ 77<< 19=%. GGKK 12::>TVT^T^A__ `hh|i/0 1Y& ($)) 2 & " &(WW__5OQ\_`Q`%a"BHHX/@B\]]r+ctj|dddf}tj|dddf}||z }|j||j |j|j }tj |j|}tjtj|gdd}|S)z%Compute binned relative position biasN)rrr)rrrrr6) r8rangerrrrgatherr expand_dims transpose)r& query_length key_lengthcontext_positionmemory_positionrrelative_position_bucketvaluess r* compute_biaszTFT5Attention.compute_biasCs88L1!T':((:.tQw7+.>>#'#A#A #.;;== $B$    ( (*B  LL +!  r+c Pt|dd\} | } |:t|dk(sJdt|d| |t|ddn|z } || n t|d} fdfd}fd }j|}||j|||dnd}||j|||dnd}j r|r||f}nd}t jd ||}|js$t jdj| | f}nj| | }|xjs|dddd| dddf}nYt jt j|ddddddfd k7}t|dd|dzdfdj| | f}|&t j||j }||z}||z }t#|d }j%|| }|ht j&j)t|jgdjdt|t j*|d|z}t j,||}j/||}|f|fz|fz}| r||fz}|S)z Self-attention (if key_value_states is None) or attention over source sentence (provided by key_value_states). Nrz?past_key_value should have 2 past states: keys and values. Got z past statesrrc tjtj|djjfdS) projectionr5rrrrperm)r8rreshaperrr= batch_sizer&s r*r/z!TFT5Attention.call..shapeus8<< =:r4<<I`I`*abiu r+cttjtj|ddjfS)zcompute contextrrr5)r8rrrrs r*unshapez#TFT5Attention.call..unshape{s.::bll=|Lz[]_c_m_mNno or+c|||}n|||}||tj||gd}|S|}|S)z4projects hidden states correctly to key/query statesrr)r8concat)r= proj_layerkey_value_statespast_key_valuer/s r*projectz#TFT5Attention.call..projectsk'!&j&? @ '!&j1A&B C )#+%'II~}.MTU$VM! %3M r+zbnqd,bnkd->bnqkgrr5rrdz/Head mask for a single layer should be of size z , but is )message)rr5rr)rlenrrrrr8einsumrzerosrr reduce_maxrrrrrr[ debugging assert_equalrmatmulr)r&r=maskr position_biasrlayer_head_maskrrrer seq_lengthreal_seq_lengthrrr query_states key_states value_statespresent_key_value_statescoresmost_recently_filled_past_indexweights attn_outputoutputsrr/s` @@r*r?zTFT5Attention.callVs&",M!:2A!> J$  %N#q( sPQTUcQdPeeqr s( BAFDLL*oF%M  " TZZ H - 4 - b1,,w,:  & LL % %?+Et||EUV"?346 & jj-@7JGii6 ffW[12 .$;#==@PP  *Gr+rkrl)T ) NNNNNNFFF)r@rArB itertoolscountrr#r2r staticmethodrrr?rCrDs@r*rrs^ Y__ F."`;0"* * X,Dr+rc>eZdZdfd Z ddZddZxZS)TFT5LayerSelfAttentionc t|di|t||d|_t |j |j d|_tjj|j|_ y)N SelfAttentionrrOr|r}r!) r"r#rrrrUrr|rrWrYrZr[r&r^rr(r)s r*r#zTFT5LayerSelfAttention.__init__sb "6"* (C   (@Y@Y`lm||++F,?,?@ r+c |j|} |j| |||||||} ||j| d|z}|f| ddz} | S)N)rrrrrrrerrdr)r|rr[) r&r=attention_maskrrrrrrerattention_outputrs r*r?zTFT5LayerSelfAttention.calls{ $}=-- '+)/.  & 5Ea5HS[ (\\  "%5ab%99r+c|jryd|_t|ddMtj|jj 5|jj ddddt|ddNtj|jj 5|jj ddddyy#1swYexYw#1swYyxYw)NTrr|)rgrhr8rirrOr2r|rjs r*r2zTFT5LayerSelfAttention.builds ::  4$ / ;t11667 /""((. / 4t , 8t334 ,%%d+ , , 9 / / , ,rrk)NNNNFFFrlrmrDs@r*rrs*A4 ,r+rc>eZdZfdZ ddZddZxZS)TFT5LayerCrossAttentionc t|di|t|dd|_t |j |j d|_tjj|j|_ y)NFEncDecAttentionrr|r}r!) r"r#rrrrUrr|rrWrYrZr[rs r*r#z TFT5LayerCrossAttention.__init__sb "6", (-"  (@Y@Y`lm||++F,?,?@ r+c |j|} |j| |||||||| |  } ||j| d| z}|f| ddz} | S)N) rrrrrrrrrerrdr)r|rr[)r&r=rrrrrrrrrerrrs r*r?zTFT5LayerCrossAttention.calls $}=// -'+)%/0  & 5Ea5HS[ (\\  "%5ab%99r+c|jryd|_t|ddMtj|jj 5|jj ddddt|ddNtj|jj 5|jj ddddyy#1swYexYw#1swYyxYw)NTrr|)rgrhr8rirrOr2r|rjs r*r2zTFT5LayerCrossAttention.build7s ::  4*D 1 =t33889 1$$**40 1 4t , 8t334 ,%%d+ , , 9 1 1 , ,r)NNNNNFFFrlrmrDs@r*rrs-A< ,r+rcFeZdZdfd Z ddZddZxZS) TFT5Blockc ~t|di||j|_g|_|jj t ||d|jr&|jj t |d|jj t|dt|jy)Nz layer_._0rz layer_._1r{zlayer_._r!) r"r#rlayerappendrrrxrrs r*r#zTFT5Block.__init__Ds "6" ++   ",G    ?? JJ  '$  +fXc$**o=N3OPQr+c | W|jsJd|dnd} t| | k7r%td| d| dk(rdnddt| d | dd}| dd}nd \}}|jd |||||| | | }|dd\}}|dd}|jrN|L|t |d d}nd}|jd |||||||| | |  }|d }|||d z}||ddz}|jd|| }|f}||fz|z}|S)Nz&Only decoder can use `past_key_values`rzThere should be z past states. z#2 (key / value) for cross attentionz. Got z past key / value states)NNr)rrrrrrrer) rrrrrrrrrer5rd)rr ValueErrorr r)r&r=rrencoder_hidden_statesencoder_attention_maskencoder_decoder_position_biasrencoder_layer_head_maskrrrreexpected_num_past_key_valuesself_attn_past_key_valuecross_attn_past_key_valueself_attention_outputsrattention_outputsrcross_attention_outputsrs r*r?zTFT5Block.callYs  %?? L$L L?0E0M1ST (>"&BB &'C&DN@\`a@a+/??ZN*>*:.HXXvw  "$Y/K 9r;r?.CDI  &$]3CR8K+/??ZN:>:J-XfWggtuv v  $$0 p2p p0 *9d6G6G6Q6Q R --i8M!, JBQA\Jq)!, -a 0: =bl   !WWj/%BAFN ??5=BWBc!+,A!B1!E %'WWj:L-Mq%Q "  "#fs4::6O}7J7JK"%j&@"A "a '&4Qa]&C # $ )((?3 mmGGGD$M2ZRS4TUD!TM* !ggk9M9MN *5aq!m*D~VWY]_cefVfGg*g'"1%1.EaZKLZ[F[.\+*8D$9I*J'$')@#@D"H ??5A&(WW-CKbKhKh%i ".1*=S2T.U +.!32HDRSUV2W/.!32HDRVXYIY2Z/035T/TX\.\ +.2 +)2t2D "6BD0d&7DOOrRV (,% ]X F 3ei-#"3! M"6C2A5F 2M2 *!,M#8#D0=CTaZ[0\-'2yT__+CG^F`+`( !/=3C2E!E??+?=QRCSBU+U(K% VN--m<  ]X F  1]4D D $&GT__!%=$??#!%6$88 !^$55??355N ??>"/ 8/)!5  %"//) r+c|jryd|_t|ddMtj|jj 5|jj ddddt|ddK|jD];}tj|j 5|j dddd=yy#1swYbxYw#1swYUxYw)NTr"r()rgrhr8rir"rOr2r()r&r3r s r*r2zTFT5MainLayer.builds ::  4+T 2 >t4499: 2%%++D1 2 4$ ' 3 &]]5::.&KK%&& & 4 2 2&&s C*CCC rl) NNNNNNNNNNNNF)returnr) r@rArBr config_classr#r,rr?r2rCrDs@r*r r scLA.""#!} }}~ &r+r c0eZdZdZeZdZdgZdZdZ dZ y)TFT5PreTrainedModelz An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained models. transformerzIdecoder\Wblock[\W_0]+layer[\W_1]+EncDecAttention\Wrelative_attention_biasc|jSrl)sharedr&s r*get_input_embeddingsz(TFT5PreTrainedModel.get_input_embeddingss {{r+c||_|j|j_t|dr|j|j_yy)Ndecoder)rVencoderr$rrZ)r&values r*set_input_embeddingsz(TFT5PreTrainedModel.set_input_embeddingss8 $(KK ! 4 #(, DLL % $r+c |jj}|jj}|Jdtjt |ddf|}tj ||j}tj||ddddfgd}|Jdtj|dk(tj tjt |||j|}tjj|tjd|j}tj|g5tj|}ddd|S#1swY|SxYw)Nzself.model.config.decoder_start_token_id has to be defined. In TF T5 it is usually set to the pad_token_id. See T5 docs for more informationrrr5z1self.model.config.pad_token_id has to be defined.ir)r^decoder_start_token_id pad_token_idr8r7rrrrrrassert_greater_equalconstantcontrol_dependenciesidentity)r&r<r_r` start_tokensshifted_input_ids assert_gte0s r* _shift_rightz TFT5PreTrainedModel._shift_rightsS!%!C!C{{// %1  > 1 ww 9 5a 8!<>TU ww|Y__= II|Yq#2#v5F&GL'\)\\'HH  % GGBGGJ'89<HJ[JaJa b  ll77 r{{14E4K4KL  $ $k] 3 ? " ,= >  ?!  ?! s E..E8N) r@rArB__doc__rrQbase_model_prefix"_keys_to_ignore_on_load_unexpectedrXr]rhr!r+r*rSrSs, L%*v)w&4 !r+rSa The T5 model was proposed in [Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer](https://arxiv.org/abs/1910.10683) by Colin Raffel, Noam Shazeer, Adam Roberts, Katherine Lee, Sharan Narang, Michael Matena, Yanqi Zhou, Wei Li, Peter J. Liu. It's an encoder decoder transformer pre-trained in a text-to-text denoising generative setting. This model inherits from [`TFPreTrainedModel`]. Check the superclass documentation for the generic methods the library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads etc.) This model is also a [keras.Model](https://www.tensorflow.org/api_docs/python/tf/keras/Model) subclass. Use it as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage and behavior. TensorFlow models and layers in `transformers` accept two formats as input: - having all inputs as keyword arguments (like PyTorch models), or - having all inputs as a list, tuple or dict in the first positional argument. The reason the second format is supported is that Keras methods prefer this format when passing inputs to models and layers. Because of this support, when using methods like `model.fit()` things should "just work" for you - just pass your inputs and labels in any format that `model.fit()` supports! If, however, you want to use the second format outside of Keras methods like `fit()` and `predict()`, such as when creating your own layers or models with the Keras `Functional` API, there are three possibilities you can use to gather all the input Tensors in the first positional argument: - a single Tensor with `input_ids` only and nothing else: `model(input_ids)` - a list of varying length with one or several input Tensors IN THE ORDER given in the docstring: `model([input_ids, attention_mask])` or `model([input_ids, attention_mask, token_type_ids])` - a dictionary with one or several input Tensors associated to the input names given in the docstring: `model({"input_ids": input_ids, "token_type_ids": token_type_ids})` Note that when creating models and layers with [subclassing](https://keras.io/guides/making_new_layers_and_models_via_subclassing/) then you don't need to worry about any of this, as you can just pass inputs like you would to any other Python function! Parameters: config ([`T5Config`]): Model configuration class with all the parameters of the model. Initializing with a config file does not load the weights associated with the model, only the configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. a Args: input_ids (`tf.Tensor` of shape `(batch_size, sequence_length)`): Indices of input sequence tokens in the vocabulary. T5 is a model with relative position embeddings so you should be able to pad the inputs on the right or the left. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.__call__`] and [`PreTrainedTokenizer.encode`] for details. [What are input IDs?](../glossary#input-ids) To know more on how to prepare `inputs` for pretraining take a look at [T5 Training](./t5#training). decoder_input_ids (`tf.Tensor` of shape `(batch_size, target_sequence_length)`, *optional*): Provide for sequence to sequence training. T5 uses the `pad_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`). To know more on how to prepare `decoder_input_ids` for pretraining take a look at [T5 Training](./t5#training). attention_mask (`tf.Tensor` of shape `(batch_size, sequence_length)`, *optional*): Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. [What are attention masks?](../glossary#attention-mask) decoder_attention_mask (`tf.Tensor` of shape `(batch_size, target_sequence_length)`, *optional*): Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also be used by default. head_mask (`tf.Tensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): Mask to nullify selected heads of the self-attention modules in the encoder. Mask values selected in `[0, 1]`: - 1 indicates the head is **not masked**, - 0 indicates the head is **masked**. decoder_head_mask (`tf.Tensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): Mask to nullify selected heads of the self-attention modules in the decoder. Mask values selected in `[0, 1]`: - 1 indicates the head is **not masked**, - 0 indicates the head is **masked**. encoder_outputs (`tuple(tuple(tf.FloatTensor)`, *optional*): Tuple consists of (`last_hidden_state`, `optional`: *hidden_states*, `optional`: *attentions*) `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)` is a sequence of hidden states at the output of the last layer of the encoder. Used in the cross-attention of the decoder. past_key_values (`tuple(tuple(tf.Tensor))` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding. If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `decoder_input_ids` of shape `(batch_size, sequence_length)`. inputs_embeds (`tf.Tensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert `input_ids` indices into associated vectors than the model's internal embedding lookup matrix. decoder_inputs_embeds (`tf.Tensor` of shape `(batch_size, target_sequence_length, hidden_size)`, *optional*): Optionally, instead of passing `decoder_input_ids` you can choose to directly pass an embedded representation. If `past_key_values` is used, optionally only the last `decoder_inputs_embeds` have to be input (see `past_key_values`). This is useful if you want more control over how to convert `decoder_input_ids` indices into associated vectors than the model's internal embedding lookup matrix. If `decoder_input_ids` and `decoder_inputs_embeds` are both unset, `decoder_inputs_embeds` takes the value of `inputs_embeds`. use_cache (`bool`, *optional*, defaults to `True`): If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see `past_key_values`). output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. This argument can be used only in eager mode, in graph mode the value in the config will be used instead. output_hidden_states (`bool`, *optional*): Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for more detail. This argument can be used only in eager mode, in graph mode the value in the config will be used instead. return_dict (`bool`, *optional*): Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. This argument can be used in eager mode, in graph mode the value will always be set to True. training (`bool`, *optional*, defaults to `False`): Whether or not to use the model in training mode (some modules like dropout modules have different behaviors between training and evaluation). aG Args: inputs (`tf.Tensor` of shape `(batch_size, sequence_length)`): Indices of input sequence tokens in the vocabulary. T5 is a model with relative position embeddings so you should be able to pad the inputs on the right or the left. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.__call__`] and [`PreTrainedTokenizer.encode`] for details. To know more on how to prepare `inputs` for pre-training take a look at [T5 Training](./t5#training). attention_mask (`tf.Tensor` of shape `(batch_size, sequence_length)`, *optional*): Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. [What are attention masks?](../glossary#attention-mask) inputs_embeds (`tf.Tensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert `input_ids` indices into associated vectors than the model's internal embedding lookup matrix. head_mask (`tf.Tensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - 1 indicates the head is **not masked**, - 0 indicates the head is **masked**. output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. output_hidden_states (`bool`, *optional*): Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for more detail. return_dict (`bool`, *optional*): Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. training (`bool`, *optional*, defaults to `False`): Whether or not to use the model in training mode (some modules like dropout modules have different behaviors between training and evaluation). a^ The input argument `head_mask` was split into two arguments `head_mask` and `decoder_head_mask`. Currently, `decoder_head_mask` is set to copy `head_mask`, but this feature is deprecated and will be removed in future versions. If you do not want to use any `decoder_head_mask` now, please set `decoder_head_mask = tf.ones((num_layers, num_heads))`. z[The bare T5 Model transformer outputting raw hidden-stateswithout any specific head on top.ceZdZfdZdZdZeeee e e  d ddZ d dZ xZS) TFT5Modelc.t||g|i|tjj |j |j tjj|jjd|_ d|j_ tj|}d|_t!||jd|_tj|}d|_|j&|_t!||jd|_y)NrV)r6 output_dimembeddings_initializerrOFr[r{TrZ)r"r#rrW Embedding vocab_sizerUrRTruncatedNormalr^rTrVload_weight_prefixcopydeepcopyrr r[rnum_decoder_layersr%rZ)r&r^inputsr(encoder_configdecoder_configr)s r*r#zTFT5Model.__init__s 3&3F3ll,,''~~#(#5#5#E#EdkkFdFd#e -  *2 &v.#( $^T[[yQ v.$(!$*$=$=!$^T[[yQ r+c|jSrlr[rWs r* get_encoderzTFT5Model.get_encoder ||r+c|jSrlrZrWs r* get_decoderzTFT5Model.get_decoderr~r+ output_typerQc|"| tjtt|}||j ||dd| |dd| | || }|d}|j ||||| |||| | | || }| r|dnd}|s||dd|fz|ddz}||zSt |j||j|j|j|j|j|jS) a Returns: Examples: ```python >>> from transformers import AutoTokenizer, TFT5Model >>> tokenizer = AutoTokenizer.from_pretrained("google-t5/t5-small") >>> model = TFT5Model.from_pretrained("google-t5/t5-small") >>> input_ids = tokenizer( ... "Studies have been shown that owning a dog is good for you", return_tensors="tf" ... ).input_ids # Batch size 1 >>> decoder_input_ids = tokenizer("Studies show that", return_tensors="tf").input_ids # Batch size 1 >>> # preprocess: Prepend decoder_input_ids with start token which is pad token for T5Model. >>> # This is not needed for torch's T5ForConditionalGeneration as it does this internally using labels arg. >>> decoder_input_ids = model._shift_right(decoder_input_ids) >>> # forward pass >>> outputs = model(input_ids, decoder_input_ids=decoder_input_ids) >>> last_hidden_states = outputs.last_hidden_state ```NF rrrr/r=r2rrr#r?rer) rrrr/r=r>r2rrr#r?rerr)r1r2decoder_hidden_statesdecoder_attentionsr4encoder_last_hidden_staterencoder_attentions) warningswarn_HEAD_MASK_WARNING_MSG FutureWarningr[rZr r1r=r3r4)r&r<rdecoder_input_idsdecoder_attention_maskr=decoder_head_maskencoder_outputsr2r/decoder_inputs_embedsrrr#r?rer=decoder_outputspasts r*r?zTFT5Model.callsA\  %6%> MM0- @ )   ""ll-&*'++# $"3%9'!+ O(* ,, 1"/#1/''+/!5#' &/q!D*"1"1"5"?/RSRTBU"U"_4 4#-?? "1"?"?.99,==&5&G&G"1"?"?.99  r+c|jryd|_tj|jjdz|jj zdz5|jj ddddt|ddMtj|jj 5|jj ddddt|ddNtj|jj 5|jj ddddyy#1swYxYw#1swYqxYw#1swYyxYw)NT/r[rZ) rgr8rirVrtrOr2rhr[rZrjs r*r2zTFT5Model.builds ::  ]]4;;99C?$++BRBRRUXX Y $ KK  d # $ 4D ) 5t||001 ) ""4( ) 4D ) 5t||001 ) ""4( ) ) 6  $ $ ) ) ) )s$D55EE 5D>E  E)NNNNNNNNNNNNNNF) r<TFModelInputType | Nonernp.ndarray | tf.Tensor | Nonerrrrr=rrrrrr24Optional[Tuple[Tuple[Union[np.ndarray, tf.Tensor]]]]r/rrrrOptional[bool]rrr#rr?rrerrPz"Union[Tuple, TFSeq2SeqModelOutput]rl)r@rArBr#r}rrrT5_INPUTS_DOCSTRINGrr _CONFIG_FOR_DOCr?r2rCrDs@r*rmrms- R**+>?+?o^.28<;?@D37;?9=PT7;?C$(,0/3&*#(!b *b 6b 9 b !> b 1 b 9b 7b Nb 5b  =b "b *b -b $b !!b " ,#b _@b H)r+rmz0T5 Model with a `language modeling` head on top.ceZdZfdZdZdZdZdZee e e e e  d d dZdZ dd Zdd Zdd ZxZS)TFT5ForConditionalGenerationct||g|i||j|_tj j |j|jdt|jj|_ d|j_ tj|}d|_t!||jd|_tj|}d|_|j&|_t!||jd|_|j,s]tj.j1d|j }tj j3|jdd | |_||_ y) NrVrOrpFr[r{TrZrrIlm_headrM)r"r#rU model_dimrrWrqrrrr^rTrVrtrurvrr r[rrwr%rZtie_word_embeddingsrRrSrXr)r&r^rxr(ryrzlm_head_initializerr)s r*r#z%TFT5ForConditionalGeneration.__init__+s5 3&3F3ll,,    NN#24;;3Q3Q#R -  *2 &v.#( $^T[[yQ v.$(!$*$=$=!$^T[[yQ ))"'"4"4"A"AqQWQjQj"A"k  <<--!!E Vi.DL r+c|jjr|jStj|j j Srl)r^rrXr8rrkernelrWs r*get_output_embeddingsz2TFT5ForConditionalGeneration.get_output_embeddingsGs9 ;; * *,,. .<< 3 34 4r+cr|jjr|j|ytjj d|jj }tjjt|ddd||_ tj|}||j_ y)NrrIFrrM)r^rr]rrRrSrTrWrXrrr8rr)r&r\rtransposed_values r*set_output_embeddingsz2TFT5ForConditionalGeneration.set_output_embeddingsOs ;; * *  % %e ,"'"4"4"A"AqQUQ\Q\QoQo"A"p  <<--5!!$u9Yl.DL "||E2 "2DLL r+c|jSrlr|rWs r*r}z(TFT5ForConditionalGeneration.get_encoder\r~r+c|jSrlrrWs r*rz(TFT5ForConditionalGeneration.get_decoder_r~r+rc|"| tjtt|}||j ||| || |||}|d}| || |j | }|j ||||| ||| | ||| }|d}|jjr?||jdzz}tj||jjd}n|j|}tj|tj }| dn|j#| |}| r|dnd}|s'||dd|fz|d dz}|f|ddz|z}||f|zS|St%|t&r1|d}d}d}d}|r |dz }||}| r |dz }||}t)||| }t+||||j,|j.|j0|j2|j,|j. S) a  labels (`tf.Tensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the cross entropy classification loss. Indices should be in `[0, ..., config.vocab_size - 1]`. Returns: Examples: ```python >>> from transformers import AutoTokenizer, TFT5ForConditionalGeneration >>> tokenizer = AutoTokenizer.from_pretrained("google-t5/t5-small") >>> model = TFT5ForConditionalGeneration.from_pretrained("google-t5/t5-small") >>> # training >>> inputs = tokenizer("The walks in park", return_tensors="tf").input_ids >>> labels = tokenizer(" cute dog the ", return_tensors="tf").input_ids >>> outputs = model(inputs, labels=labels) >>> loss = outputs.loss >>> logits = outputs.logits >>> # inference >>> inputs = tokenizer( ... "summarize: studies have shown that owning a dog is good for you", return_tensors="tf" ... ).input_ids # Batch size 1 >>> outputs = model.generate(inputs) >>> print(tokenizer.decode(outputs[0], skip_special_tokens=True)) >>> # studies have shown that owning a dog is good for you ```N)rr/r=rr#r?rerrrHT) transpose_brrr5) losslogitsr2rrr4rrr)rrrrr[rhrZr^rrr8rrVrrrrhf_compute_loss isinstancetupler r r=r3r4r1)r&r<rrrr=rrr2r/rlabelsrrr#r?rer=rsequence_outputrrroutputr1r3rMs r*r?z!TFT5ForConditionalGeneration.callbs`j  %6%> MM0- @ )   ""ll-+#"3%9'!+ O(*  "3";@U@] $ 1 1& 9 ,, 1"/#1/'+/!5#'  *!, ;; * *-1EFOYY 0C0CQUVF\\/2F,~t4+?+?+O%.q!D*"1"1"5"?/RSRTBU"UY!44FF)-)9TGf$ Ev E / / 2  MJC#q / 4  q,S1 /"3+%O ! "1"?"?.99,==&5&G&G"1"?"?.99  r+c |jjr"tj|jddnd}|jj rtj|j nd}|jjrtj|jnd}|jjrtj|jnd}|jj rtj|jnd}|jjrtj|jnd}t|j|||||j||S)Nr)rr2rrr4rrr)r^rr8convert_to_tensorr2r#rrrr4rrr rr)r&rpkvdec_hs dec_attns cross_attnsenc_hs enc_attnss r*serving_outputz+TFT5ForConditionalGeneration.serving_outputsBF++BWBWb""6#9#9!"#=>]aGK{{GgGg%%f&B&BCmqGK{{GdGdB(()B)BCjn GK{{GdGdb**6+B+BCjn GK{{GgGg%%f&B&BCmqGK{{GdGdB(()B)BCjn  =="(((&,&F&F"((  r+c 4| |ddddf}d||||||||d S)Nr5) r<rr2rrrr=rrr!) r&r<r2rrr=rrrr(s r*prepare_inputs_for_generationz:TFT5ForConditionalGeneration.prepare_inputs_for_generationsA  &!!RS&)I!*..,&<"!2"  r+c$|j|Srl)rh)r&rs r*%prepare_decoder_input_ids_from_labelszBTFT5ForConditionalGeneration.prepare_decoder_input_ids_from_labels s  ((r+c|jryd|_tj|jjdz|jj zdz5|jj ddddt|ddMtj|jj 5|jj ddddt|ddMtj|jj 5|jj ddddt|ddetj|jj 5|jj dd|jjgdddyy#1swY1xYw#1swYxYw#1swYxYw#1swYyxYw)NTrr[rZr) rgr8rirVrtrOr2rhr[rZrr^rUrjs r*r2z"TFT5ForConditionalGeneration.build#sp ::  ]]4;;99C?$++BRBRRUXX Y $ KK  d # $ 4D ) 5t||001 ) ""4( ) 4D ) 5t||001 ) ""4( ) 4D ) 5t||001 F ""D$ 0C0C#DE F F 6 $ $ ) ) ) ) F Fs0F&5F3F?)3G &F03F<?G G)NNNNNNNNNNNNNNNF)"r<rrrrrrrr=rrrrrr2rr/rrrrrrrrrr#rr?rrerrPzUnion[Tuple, TFSeq2SeqLMOutput])NNNNNNN)rz tf.Tensorrl)r@rArBr#rrr}rrrrrr rr?rrrr2rCrDs@r*rr)sk85 3*+>?+ J 1 J 9J 7J NJ 5J  =J .J "J *J -J $!J "!#J $ )%J \@J X ,# 8)Fr+rzeThe bare T5 Model transformer outputting encoder's raw hidden-stateswithout any specific head on top.ceZdZfdZdZeeeee e  d ddZ ddZ xZ S) TFT5EncoderModelcrt||g|i|tjj |j |j dt|jj|_ d|j_ tj|}d|_t||jd|_y)NrVrFr[r{)r"r#rrWrqrrrUrr^rTrVrtrurvrr r[)r&r^rxr(ryr)s r*r#zTFT5EncoderModel.__init__<s 3&3F3ll,,    NN#24;;3Q3Q#R -  *2 &v.#( $^T[[yQ r+c|jSrlr|rWs r*r}zTFT5EncoderModel.get_encoderKr~r+rc |j||dd||dd|||| } |s| St| j| j| jS)a Returns: Examples: ```python >>> from transformers import AutoTokenizer, TFT5EncoderModel >>> tokenizer = AutoTokenizer.from_pretrained("google-t5/t5-small") >>> model = TFT5EncoderModel.from_pretrained("google-t5/t5-small") >>> input_ids = tokenizer( ... "Studies have been shown that owning a dog is good for you", return_tensors="tf" ... ).input_ids # Batch size 1 >>> outputs = model(input_ids) ```NFrr5)r[r r1r=r3) r&r<rr=r/rr#r?rers r*r?zTFT5EncoderModel.callNsn>,, )"&#'' /!5#'  " " -??)77&11  r+c|jryd|_tj|jjdz|jj zdz5|jj ddddt|ddNtj|jj 5|jj ddddyy#1swYexYw#1swYyxYw)NTrr[) rgr8rirVrtrOr2rhr[rjs r*r2zTFT5EncoderModel.builds ::  ]]4;;99C?$++BRBRRUXX Y $ KK  d # $ 4D ) 5t||001 ) ""4( ) ) 6 $ $ ) )sC5C'C$'C0)NNNNNNNF)r<rrrr=rr/rrrr#rr?rrerrPzUnion[Tuple, TFBaseModelOutput]rl)r@rArBr#r}rrT5_ENCODER_INPUTS_DOCSTRINGrr rr?r2rCrDs@r*rr7s R*+FG+rs "  ))?/ SR '   H %+ELL&&+(#> **#>L*>U\\//*>Z0%,,$$0:RELL&&Rj.,U\\//.,b2,ell002,jm- ""m-hg&ELL&&g&g&b2!+2!j-^Rh&PaQ)#Q) Q)hLN`aJF#68TJFbJFZkU)*U) U)r+