sgdZddlmZddlZddlZddlZddlmZmZm Z m Z ddl Z ddl mZddlmZmZmZmZddlmZmZmZmZmZdd lmZmZmZdd lmZm Z m!Z!m"Z"m#Z#m$Z$d d l%m&Z&e#jNe(Z)d Z*dZ+dZ,d.dZ-d/d0dZ.d1d2dZ/Gddej`jbZ2Gddej`jfZ4Gddej`jfZ5Gddej`jfZ6GddeZ7dZ8dZ9dZ:eGd d!ej`jfZ;eGd"d#ej`jfZGd)d*ej`jfZ?e!d+e8Gd,d-e7eZ@y)3zTF 2.0 Blenderbot model.) annotationsN)ListOptionalTupleUnion)get_tf_activation)TFBaseModelOutput+TFBaseModelOutputWithPastAndCrossAttentionsTFSeq2SeqLMOutputTFSeq2SeqModelOutput)TFCausalLanguageModelingLossTFPreTrainedModelkeraskeras_serializable unpack_inputs)check_embeddings_within_bounds shape_liststable_softmax)add_code_sample_docstringsadd_end_docstringsadd_start_docstrings%add_start_docstrings_to_model_forwardloggingreplace_return_docstrings)BlenderbotConfigz facebook/blenderbot-400M-distillrgחc tj||j}tj||j}tjt |ddftj ||j}tj ||ddddfgd}tj|dk(tjt |tj ||j|}tjj|tjd|j}tj|g5tj|}ddd|S#1swY|SxYw)Nrrdtype) tfcastr"fillrconvert_to_tensorconcatwhere debuggingassert_greater_equalconstantcontrol_dependenciesidentity) input_ids pad_token_iddecoder_start_token_id start_tokensshifted_input_ids assert_gte0s h/var/www/html/venv/lib/python3.12/site-packages/transformers/models/blenderbot/modeling_tf_blenderbot.pyshift_tokens_rightr5@s877<9LWW%;Y__M77 I q !1%r';'; ::ct|d}||n|}tjd}tj||j}tj |ddddddfdd|df}||z t zS)z_ Expands attention_mask from `[bsz, seq_len]` to `[bsz, 1, tgt_seq_len, src_seq_len]`. rN?r!)rr#r+r$r"r?r;)rDrCsrc_lenone_cst expanded_masks r4 _expand_maskrMksxq!G ,g'Gkk#G 774w}} -DGGDD$!12Q7A4FGM m #~ 55rGc>eZdZdZdfd Z d dfd ZxZS)&TFBlenderbotLearnedPositionalEmbeddingzN This module learns positional embeddings up to a fixed maximum size. c (t|||fi|yN)super__init__)selfnum_embeddings embedding_dimkwargs __class__s r4rSz/TFBlenderbotLearnedPositionalEmbedding.__init__}s A&ArGc|"|d}tj|dd}||z }t| tj|tj S)z/Input is expected to be of size [bsz x seqlen].rr<)deltanamer!)r#r<rRcallr$int32)rT input_shaperA position_idsseq_lenrXs r4r\z+TFBlenderbotLearnedPositionalEmbedding.callsO  !!nG88G17CL 2 2Lw|BGGLABBrG)rUintrVra)rN)r^tf.TensorShaperArar_tf.Tensor | None__name__ __module__ __qualname____doc__rSr\ __classcell__rXs@r4rOrOxs<Bnr C) CCF CZj C CrGrOc|eZdZdZ d dfd ZddZ d d dZd dZxZS) TFBlenderbotAttentionz6Multi-headed attention from "Attention Is All You Needc zt|d i|||_||_tj j ||_||z|_|j|z|jk7rtd|jd|d|jdz|_ ||_ tj j||d|_ tj j||d|_tj j||d|_tj j||d |_y) Nz;embed_dim must be divisible by num_heads (got `embed_dim`: z and `num_heads`: z).gk_proj)use_biasr[q_projv_projout_proj)rRrS embed_dim num_headsrlayersDropoutdropouthead_dim ValueErrorscaling is_decoderDensernrprqrr)rTrtrurxr|biasrWrXs r4rSzTFBlenderbotAttention.__init__s "6"""||++G4 !Y. MMI %$.. 8MdnnM]$YKr3 }}d* $ll((T(Q ll((T(Q ll((T(Q  **9t**U rGc tjtj||||j|jfdS)Nrrr)r# transposer=rury)rTtensorr`rBs r4_shapezTFBlenderbotAttention._shapes0||BJJvWdnndmm/\]_kllrGc  |du}t|\}} } |j||jz} |r| |d} |d} n |rE|j|j |d|} |j|j |d|} n|}|j|j |d|} |j|j |d|} t j|d| gd} t j|d| gd} nD|j|j |d|} |j|j |d|} |jr| | f}||jzd|jf}t j|j| | ||} t j| |} t j| |} t| d}t j| | d}t jjt|||jz| |gd ||jz| |fd t| |t jjt||d| |gd |d| |fd t| t j||j }t j|||j| |f|z}t j|||jz| |f}t#|d}|t jjt||jgd|jd t| t j|dt j|||j| |fz}t j|||jz| |f}|j%||}t j|| }t jjt|||jz| |jgd||j| |jfd t| t j&t j|||j| |jfd}t j||| | f}|j)|}t j|||j| |f}|||fS)z#Input shape: Batch x Time x ChannelNrrrrr8T transpose_bz$Attention weights should be of size z , but is messagez!Attention mask should be of size r!z/Head mask for a single layer should be of size )rrrrtrainingz `attn_output` should be of size r)rrpr{rrnrqr#r'r|ruryr=matmulr) assert_equalr$r"rrxrrr)rT hidden_stateskey_value_statespast_key_valueattention_masklayer_head_maskris_cross_attentionrBrCrt query_states key_states value_states proj_shaperJ attn_weights attn_probs attn_outputs r4r\zTFBlenderbotAttention.calls.T9",]";Wi{{=1DLL@ ."<'*J)!,L T[[1A%BBLJ;;t{{3C'Db#NL  'T[[%?SIJ;;t{{='A2sKLN1$5z#BKJ99nQ&7%FQOLT[[%?SIJ;;t{{='A2sKL ??),7NDNN*B > zz$++lGS"I:V ZZ J7 zz, ; Z(+yyztL  !! | $ 4>> !7G 46dnn8LgW^7_6`a|,-/ "   % LL % %>*a'*7a'8R7ST">235 &  WW^<;M;MNN::lS$..'SZ4[\_mmL::lS4>>5I7T[4\]L%l<  & LL % %?+ Et~~EWX"?346 & ::o}E sDNNGWEIL::lS4>>5I7T[4\]L\\,\B ii L9  !! { # 4>> !7DMM :2CRVR_R_3`2ab{+,. " ll JJ{S$..'4==$Q RT` jjsGY.GH mmK0 "$**\CQXZa;b"c L.88rGc|jryd|_t|ddZtj|jj 5|jj dd|jgdddt|ddZtj|jj 5|jj dd|jgdddt|ddZtj|jj 5|jj dd|jgdddt|dd[tj|jj 5|jj dd|jgdddyy#1swYAxYw#1swYxYw#1swYxYw#1swYyxYw)NTrnrprqrr) builtgetattrr# name_scopernr[buildrtrprqrrrTr^s r4rzTFBlenderbotAttention.build%s ::  44 ( 4t{{//0 @ !!4t~~">? @ 44 ( 4t{{//0 @ !!4t~~">? @ 44 ( 4t{{//0 @ !!4t~~">? @ 4T * 6t}}112 B ##T4$@A B B 7 @ @ @ @ @ @ B Bs0 )F32)G)G )G3F=G  GG!)r7FT) rtrarurarxfloatr|boolr~r)r tf.Tensorr`rarBra)NNNNF)rrrrcrzTuple[Tuple[tf.Tensor]] | NonerrcrrcrOptional[bool]returnz"Tuple[tf.Tensor, tf.Tensor | None]rQ) rerfrgrhrSrr\rrirjs@r4rlrls@   VVV V  V  V8m .29=+/,0#(t9 t9+t97 t9 ) t9 * t9!t9 ,t9lBrGrlcBeZdZdfd Z d ddZddZxZS)TFBlenderbotEncoderLayerc t|d i||j|_t |j|j |j d|_tjjdd|_ tjj|j|_ t|j|_tjj|j"|_tjj%|j&d|_tjj%|jd|_tjjdd |_||_y) N self_attn)rxr[h㈵>self_attn_layer_normepsilonr[fc1r[fc2final_layer_normrs)rRrSd_modelrtrlencoder_attention_headsattention_dropoutrrrvLayerNormalizationrrwrxr activation_function activation_fnactivation_dropoutr}encoder_ffn_dimrrrconfigrTrrWrXs r4rSz!TFBlenderbotEncoderLayer.__init__9s "6". NNF::FD\D\cn %*LL$C$CDWm$C$n!||++FNN; .v/I/IJ"',,"6"6v7P7P"Q<<%%f&<&<5%I<<%%dnn5%A % ? ?Se ? f rGc |}|j|}|j|||\}}}tjj t |t |dt |dt ||j ||}||z}|}|j|}|j|j|}|j||}|j|}|j ||}||z}||fS)a Args: hidden_states (`tf.Tensor`): input to the layer of shape *(batch, seq_len, embed_dim)* attention_mask (`tf.Tensor`): attention mask of size *(batch, 1, tgt_len, src_len)* where padding elements are indicated by very large negative values. layer_head_mask (`tf.Tensor`): mask for attention heads in a given layer of size *(encoder_attention_heads,)* )rrrz&Self attn modified the shape of query z to rr) rrr#r)rrrxrrrrr)rTrrrrresidualself_attn_weights_s r4r\zTFBlenderbotEncoderLayer.callHs!11-@ .2nn'Xg/=/ + (! !! } % x  N))//tT^^0LM N 4 % 1txx}}- =dDNN;< = 4 % 1txx}}- JdDKK,G,GHI J 4+T 2 >t4499: J%%++T4,HI J J ? + + N N = = J J J Js< H%)H$ )H133H=$)I H!$H.1H:=I Irr)F)rrrrrrrrrQrerfrgrSr\rrirjs@r4rr8s? ($) &0 &0"&0# &0 ! &0PJrGrcbeZdZdfd Z d ddZddZxZS)TFBlenderbotDecoderLayerc t|di||j|_t |j|j |j dd|_tjj|j|_ t|j|_tjj|j|_tjj!dd|_t |j|j |j dd|_tjj!dd |_tjj)|j*d |_tjj)|jd |_tjj!dd |_||_y)NrT)rtrurxr[r|rrr encoder_attn)rxr[r|encoder_attn_layer_normrrrrrs)rRrSrrtrldecoder_attention_headsrrrrvrwrxr rrrrrrrr}decoder_ffn_dimrrrrrs r4rSz!TFBlenderbotDecoderLayer.__init__sa "6".nn44,,  ||++FNN; .v/I/IJ"',,"6"6v7P7P"Q$)LL$C$CDWm$C$n!1 NN  * *,,  (-||'F'FtZs'F't$<<%%f&<&<5%I<<%%dnn5%A % ? ?Se ? f rGc 8|} |j|}||ddnd} |j|| ||\}} } |j||}| |z}d} d}|S|} |j|}||ddnd}|j |||||\}}} |j||}| |z}| | z} |} |j |}|j |j|}|j||}|j|}|j||}| |z}|| || fS)a Args: hidden_states (`tf.Tensor`): input to the layer of shape *(batch, seq_len, embed_dim)* attention_mask (`tf.Tensor`): attention mask of size *(batch, 1, tgt_len, src_len)* where padding elements are indicated by very large negative values. encoder_hidden_states (`tf.Tensor`): cross attention input to the layer of shape *(batch, seq_len, embed_dim)* encoder_attention_mask (`tf.Tensor`): encoder attention mask of size *(batch, 1, tgt_len, src_len)* where padding elements are indicated by very large negative values. layer_head_mask (`tf.Tensor`): mask for attention heads in a given layer of size *(decoder_attention_heads,)* cross_attn_layer_head_mask (`tf.Tensor`): mask for heads of the cross-attention module. *(decoder_attention_heads,)* past_key_value (`Tuple(tf.Tensor)`): cached past key and value projection states Nr)rrrrr)rrrrr) rrrxrrrrrrr)rTrrencoder_hidden_statesencoder_attention_maskrcross_attn_layer_head_maskrrrself_attn_past_key_valuerpresent_key_valuecross_attn_present_key_valuecross_attn_weightscross_attn_past_key_values r4r\zTFBlenderbotDecoderLayer.calls4!11-@ :H9S>"1#5Y] >Bnn'3)+ ?M? ; (*;  ]X F  =0 (,$! ,$H 88GM@N?Yrs(;_c %NRN_N_+!65 :8 O`O KM-/K!LLLJM$}4M!24P P !--m< **488M+BC // /Q /  ]X F  =0        rGcb|jryd|_t|ddMtj|jj 5|jj ddddt|ddZtj|jj 5|jj dd|jgdddt|ddMtj|jj 5|jj ddddt|ddZtj|jj 5|jj dd|jgdddt|ddZtj|jj 5|jj dd|jgdddt|dddtj|jj 5|jj dd|jjgdddt|dd[tj|jj 5|jj dd|jgdddyy#1swYsxYw#1swYxYw#1swYxYw#1swYrxYw#1swYxYw#1swYxYw#1swYyxYw) NTrrrrrrr)rrr#rrr[rrrtrrrrrrrrs r4rzTFBlenderbotDecoderLayer.builds ::  4d + 7t~~223 +$$T* + 4/ 6 Bt88==> N))//tT^^0LM N 4 . :t00556 .!!''- . 42D 9 Et;;@@A Q,,22D$3OP Q 4 % 1txx}}- =dDNN;< = 4 % 1txx}}- JdDKK,G,GHI J 4+T 2 >t4499: J%%++T4,HI J J ?# + + N N . . Q Q = = J J J JsT K%)K% K2&)K? )L 43L%)L%K"%K/2K<?L  LL"%L.r)NNNNNNF)rrrrcrrcrrcrrcrrcrzTuple[tf.Tensor] | Nonerrrz4Tuple[tf.Tensor, tf.Tensor, Tuple[Tuple[tf.Tensor]]]rQrrjs@r4rrs>,02637,07;26#(N  N )N  0 N !1 N * N %5N 0N !N  >N `JrGrceZdZeZdZy)TFBlenderbotPreTrainedModelmodelN)rerfrgr config_classbase_model_prefixrsrGr4rrs #LrGra{ This model inherits from [`TFPreTrainedModel`]. Check the superclass documentation for the generic methods the library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads etc.) This model is also a [keras.Model](https://www.tensorflow.org/api_docs/python/tf/keras/Model) subclass. Use it as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage and behavior. TensorFlow models and layers in `transformers` accept two formats as input: - having all inputs as keyword arguments (like PyTorch models), or - having all inputs as a list, tuple or dict in the first positional argument. The reason the second format is supported is that Keras methods prefer this format when passing inputs to models and layers. Because of this support, when using methods like `model.fit()` things should "just work" for you - just pass your inputs and labels in any format that `model.fit()` supports! If, however, you want to use the second format outside of Keras methods like `fit()` and `predict()`, such as when creating your own layers or models with the Keras `Functional` API, there are three possibilities you can use to gather all the input Tensors in the first positional argument: - a single Tensor with `input_ids` only and nothing else: `model(input_ids)` - a list of varying length with one or several input Tensors IN THE ORDER given in the docstring: `model([input_ids, attention_mask])` or `model([input_ids, attention_mask, token_type_ids])` - a dictionary with one or several input Tensors associated to the input names given in the docstring: `model({"input_ids": input_ids, "token_type_ids": token_type_ids})` Note that when creating models and layers with [subclassing](https://keras.io/guides/making_new_layers_and_models_via_subclassing/) then you don't need to worry about any of this, as you can just pass inputs like you would to any other Python function! Args: config ([`BlenderbotConfig`]): Model configuration class with all the parameters of the model. Initializing with a config file does not load the weights associated with the model, only the configuration. Check out the [`~TFPreTrainedModel.from_pretrained`] method to load the model weights. au Conversation example:: ```py >>> from transformers import AutoTokenizer, TFBlenderbotForConditionalGeneration >>> mname = "facebook/blenderbot-400M-distill" >>> model = TFBlenderbotForConditionalGeneration.from_pretrained(mname) >>> tokenizer = AutoTokenizer.from_pretrained(mname) >>> UTTERANCE = "My friends are cool but they eat too many carbs." >>> print("Human: ", UTTERANCE) >>> inputs = tokenizer([UTTERANCE], return_tensors="tf") >>> reply_ids = model.generate(**inputs) >>> print("Bot: ", tokenizer.batch_decode(reply_ids, skip_special_tokens=True)[0]) >>> REPLY = "I'm not sure" >>> print("Human: ", REPLY) >>> NEXT_UTTERANCE = ( ... "My friends are cool but they eat too many carbs. That's unfortunate. " ... "Are they trying to lose weight or are they just trying to be healthier? " ... " I'm not sure." ... ) >>> inputs = tokenizer([NEXT_UTTERANCE], return_tensors="tf") >>> next_reply_ids = model.generate(**inputs) >>> print("Bot: ", tokenizer.batch_decode(next_reply_ids, skip_special_tokens=True)[0]) ``` aE Args: input_ids (`tf.Tensor` of shape `({0})`): Indices of input sequence tokens in the vocabulary. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) attention_mask (`tf.Tensor` of shape `({0})`, *optional*): Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. [What are attention masks?](../glossary#attention-mask) decoder_input_ids (`tf.Tensor` of shape `(batch_size, target_sequence_length)`, *optional*): Indices of decoder input sequence tokens in the vocabulary. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are decoder input IDs?](../glossary#decoder-input-ids) Blenderbot uses the `bos_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`). decoder_attention_mask (`tf.Tensor` of shape `(batch_size, target_sequence_length)`, *optional*): will be made by default and ignore pad tokens. It is not recommended to set this for most use cases. decoder_position_ids (`tf.Tensor` of shape `(batch_size, sequence_length)`, *optional*): Indices of positions of each decoder input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`. head_mask (`tf.Tensor` of shape `(encoder_layers, encoder_attention_heads)`, *optional*): Mask to nullify selected heads of the attention modules in the encoder. Mask values selected in `[0, 1]`: - 1 indicates the head is **not masked**, - 0 indicates the head is **masked**. decoder_head_mask (`tf.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): Mask to nullify selected heads of the attention modules in the decoder. Mask values selected in `[0, 1]`: - 1 indicates the head is **not masked**, - 0 indicates the head is **masked**. cross_attn_head_mask (`tf.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): Mask to nullify selected heads of the cross-attention modules. Mask values selected in `[0, 1]`: - 1 indicates the head is **not masked**, - 0 indicates the head is **masked**. encoder_outputs (`tf.FloatTensor`, *optional*): hidden states at the output of the last layer of the encoder. Used in the cross-attention of the decoder. of shape `(batch_size, sequence_length, hidden_size)` is a sequence of past_key_values (`Tuple[Tuple[tf.Tensor]]` of length `config.n_layers`) contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding. If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `decoder_input_ids` of shape `(batch_size, sequence_length)`. use_cache (`bool`, *optional*, defaults to `True`): If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see `past_key_values`). Set to `False` during training, `True` during generation output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. This argument can be used only in eager mode, in graph mode the value in the config will be used instead. output_hidden_states (`bool`, *optional*): Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for more detail. This argument can be used only in eager mode, in graph mode the value in the config will be used instead. return_dict (`bool`, *optional*): Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. This argument can be used in eager mode, in graph mode the value will always be set to True. training (`bool`, *optional*, defaults to `False`): Whether or not to use the model in training mode (some modules like dropout modules have different behaviors between training and evaluation). c^eZdZeZ ddfd ZdZdZe ddZ ddZ xZ S) TFBlenderbotEncoderc t|di|||_tjj |j |_|j|_|j|_ |j|_ |jr2tjj!t#|j$nd|_||_t+|j|j$d|_t/|j0Dcgc]}t3|d|c}|_tjj5dd|_ycc}w) NrIembed_positionsrlayers.r layer_normrrs)rRrSrrrvrwrxencoder_layerdrop layerdropr/ padding_idxmax_position_embeddingsmax_source_positionsscale_embeddingr#mathsqrtrr embed_scale embed_tokensrOrr<encoder_layersrrrrTrrrWirXs r4rSzTFBlenderbotEncoder.__init__s "6" ||++FNN; 11!..$*$B$B!BHBXBX277<<fnn(=>^a(E  * * NN"  V[[a[p[pUqrPQ/wqc]Kr ,,99$\9ZssE c|jSrQrrTs r4get_embed_tokensz$TFBlenderbotEncoder.get_embed_tokens   rGc||_yrQrrTrs r4set_embed_tokensz$TFBlenderbotEncoder.set_embed_tokens (rGc | | td| t|} n|t|dd} n td|>t||jj|j||j z}|j | } || z} |j| |} | t|}nd}|rdnd} |rdnd} |gtjjt|dt|jdt|jd t|dd  t|jD]R\}}|r| | fz} tj dd }|r||j"kr6|| ||||nd\} }|sM| |fz } T|j%| } |r| | fz} |st'd | | | fDSt)| | | S)a Args: input_ids (`tf.Tensor` of shape `(batch_size, sequence_length)`): Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) attention_mask (`tf.Tensor` of shape `(batch_size, sequence_length)`, *optional*): Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. [What are attention masks?](../glossary#attention-mask) head_mask (`tf.Tensor` of shape `(encoder_layers, encoder_attention_heads)`, `optional): Mask to nullify selected heads of the attention modules. Mask values selected in `[0, 1]`: - 1 indicates the head is **not masked**, - 0 indicates the head is **masked**. inputs_embeds (`tf.Tensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert `input_ids` indices into associated vectors than the model's internal embedding lookup matrix. output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. This argument can be used only in eager mode, in graph mode the value in the config will be used instead. output_hidden_states (`bool`, *optional*): Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for more detail. This argument can be used only in eager mode, in graph mode the value in the config will be used instead. return_dict (`bool`, *optional*): Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. This argument can be used in eager mode, in graph mode the value will always be set to True. training (`bool`, *optional*, defaults to `False`): Whether or not to use the model in training mode (some modules like dropout modules have different behaviors between training and evaluation). NzDYou cannot specify both input_ids and inputs_embeds at the same timerz5You have to specify either input_ids or inputs_embedsrrsrz&The head_mask should be specified for  layers, but it is for .rrc3&K|] }|| ywrQrs).0vs r4 z+TFBlenderbotEncoder.call..>seqWXWdeslast_hidden_stater attentions)rzrrr input_dimrrrxrMr#r)rlenrv enumeraterandomuniformrrtupler )rTr. inputs_embedsr head_maskoutput_attentionsoutput_hidden_states return_dictrr^ embed_posrencoder_statesall_attentionsidx encoder_layerdropout_probabilityattns r4r\zTFBlenderbotEncoder.callsl  ]%>cd d  "$Y/K  &$]3CR8KTU U  *9d6G6G6Q6Q R --i84;K;KKM((5 % 1  ]X F   %).9N!N30d  LL % %9%a(DKK >A"/"+"7 #T# M4 !4') *" 6 +}.>>Ne]NN$See e +>Vd  rGc|jryd|_t|ddMtj|jj 5|jj ddddt|dddtj|jj 5|jj dd|jjgdddt|ddK|jD];}tj|j 5|j dddd=yy#1swYxYw#1swYnxYw#1swYaxYwNTrrrv rrr#rrr[rrrrrvrTr^layers r4rzTFBlenderbotEncoder.buildC" ::  4*D 1 =t33889 1$$**40 1 4t , 8t334 I%%tT4;;3F3F&GH I 44 ( 4 &]]5::.&KK%&& & 5  1 1 I I&&$ D9%3EE9EEE rQrrrz Optional[keras.layers.Embedding])NNNNNNNF rerfrgrrrSrrrr\rrirjs@r4rrsS#L[$!)!v v p &rGrcjeZdZeZ ddfd ZdZdZe ddZ ddZ xZ S) TFBlenderbotDecoderc zt|di|||_|j|_||_|j |_t|j|jd|_ |jr2tjjt!|jnd|_t%|j&Dcgc]}t)|d|c}|_t,j*j/dd|_t,j*j3|j4|_ycc}w) NrrrIrrrrrs)rRrSrr/rrdecoder_layerdroprrOrrrrr#rrrrr<decoder_layersrrvrrrrwrxrs r4rSzTFBlenderbotDecoder.__init__^s "6" !..(11E  * * NN"  CIBXBX277<<fnn(=>^aUZ[a[p[pUqrPQ/wqc]Kr ,,99$\9Z||++FNN; ssD8c|jSrQrrs r4rz$TFBlenderbotDecoder.get_embed_tokensorrGc||_yrQrrs r4rz$TFBlenderbotDecoder.set_embed_tokensrrrGc 8| | td| t|}n|t|dd}n td| t| dddnd}||j||}n|j||}|>t||jj |j ||j z}|}|ddkDrt|| }n.ttj|d|d|zf|d }||t||d z}||t||d }||z}|j|| }| rd nd}| rd nd}| r|d nd}| rd nd}d |fd|ffD]r\}}| tjjt|dt|jd|dt|jdt|ddtt!|jD]z\}}| r||fz }t#j$dd}|r||j&kr6| | |nd}||||||||nd|||nd|\}}}} | r|| fz }| sl||fz }|u||fz }||j)|}| r||fz }| s|||||fSt+|||||S)a Args: input_ids (`tf.Tensor` of shape `(batch_size, sequence_length)`): Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) attention_mask (`tf.Tensor` of shape `(batch_size, sequence_length)`, *optional*): Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. [What are attention masks?](../glossary#attention-mask) position_ids (`tf.Tensor` of shape `(batch_size, sequence_length)`, *optional*): Indices of positions of each decoder input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`. encoder_hidden_states (`tf.Tensor` of shape `(batch_size, encoder_sequence_length, hidden_size)`, *optional*): Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention of the decoder. encoder_attention_mask (`tf.Tensor` of shape `(batch_size, encoder_sequence_length)`, *optional*): Mask to avoid performing cross-attention on padding tokens indices of encoder input_ids. Mask values selected in `[0, 1]`: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. [What are attention masks?](../glossary#attention-mask) head_mask (`tf.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): Mask to nullify selected heads of the attention modules. Mask values selected in `[0, 1]`: - 1 indicates the head is **not masked**, - 0 indicates the head is **masked**. cross_attn_head_mask (`tf.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): Mask to nullify selected heads of the cross-attention modules. Mask values selected in `[0, 1]`: - 1 indicates the head is **not masked**, - 0 indicates the head is **masked**. past_key_values (`Tuple[Tuple[tf.Tensor]]` of length `config.n_layers` with each tuple having 2 tuples each of which has 2 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): Contains precomputed key and value hidden-states of the attention blocks. Can be used to speed up decoding. If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `decoder_input_ids` of shape `(batch_size, sequence_length)`. inputs_embeds (`tf.Tensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert `input_ids` indices into associated vectors than the model's internal embedding lookup matrix. output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. This argument can be used only in eager mode, in graph mode the value in the config will be used instead. output_hidden_states (`bool`, *optional*): Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for more detail. This argument can be used only in eager mode, in graph mode the value in the config will be used instead. return_dict (`bool`, *optional*): Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. This argument can be used in eager mode, in graph mode the value will always be set to True. training (`bool`, *optional*, defaults to `False`): Whether or not to use the model in training mode (some modules like dropout modules have different behaviors between training and evaluation). NzTYou cannot specify both decoder_input_ids and decoder_inputs_embeds at the same timerzEYou have to specify either decoder_input_ids or decoder_inputs_embedsrr)r_r)rA)rCrrsrcross_attn_head_maskzThe z should be specified for rrr)rrrrrr)rpast_key_valuesrrcross_attentions)rzrrrrrrrFrMr#r:rxr)rrrvrrrrrr )!rTr.rrr_rrrr#r$ use_cacherr r rr^rA positionsrcombined_attention_maskall_hidden_statesall_self_attnsall_cross_attnspresent_key_valuesattn_mask_name attn_maskr decoder_layerrrlayer_self_attnlayer_cross_attnrs! r4r\zTFBlenderbotDecoder.callusn  ]%>st t  "$Y/K  &$]3CR8Kde eIXIdOA,>q,A!B1!Ejk  ,,[:PQI,,[|,TI  *9d6G6G6Q6Q R --i84;K;KKM%  r?Q &7 \r&s #&2QQ:P)PQR\ghj\k' #  %&= ^epqset@u&u # ,1G1S%12HR]^`Ra%b "% 1  ]X F #7BD0d!27L7X"_c#,R$,7 *BE[]qDr)s  %NI$ ))y)!, $~..GDKKHXGYZ&y1!45Q8 * #,DKK"8 ; C#!m%55!"(..A"6 04>>A5D5P_S1VZNR_6&;'=2;2G #TH\Hh+?+Dnr-S OM?,<>O"'8&::" ?"44(4#(8'::O9 ;< 6  -!1 1  "46GYhh h>"/ 2/)!0  rGc|jryd|_t|ddMtj|jj 5|jj ddddt|dddtj|jj 5|jj dd|jjgdddt|ddK|jD];}tj|j 5|j dddd=yy#1swYxYw#1swYnxYw#1swYaxYwrrrs r4rzTFBlenderbotDecoder.build4rrrQr)NNNNNNNNNNNNNFrrjs@r4rrSsd#L<"!)"#!!||| &rGrcpeZdZeZdfd ZdZdZe d ddZ d dZ xZ S) TFBlenderbotMainLayerc t|di|||_tjj |j |jtjj|jjd|_ d|j_ t||jd|_t||jd|_y)N)stddevz model.shared)r output_dimembeddings_initializerr[encoderrdecoderrs)rRrSrrrv Embedding vocab_sizer initializersTruncatedNormalinit_stdsharedload_weight_prefixrr9rr:rs r4rSzTFBlenderbotMainLayer.__init__Hs "6" ll,,''~~#(#5#5#E#ET[[MaMa#E#b -  *8 &*64;;YO *64;;YO rGc|jSrQ)r@rs r4get_input_embeddingsz*TFBlenderbotMainLayer.get_input_embeddingsXs {{rGc~||_|j|j_|j|j_yrQ)r@r9rr:)rTnew_embeddingss r4set_input_embeddingsz*TFBlenderbotMainLayer.set_input_embeddings[s)$ $(KK !$(KK !rGc ^||n|jj}| |j|||| ||||} nl|rHt| ts8t | dt | dkDr| dndt | dkDr| dnd} n"|s t| t s| j} |j|||| d|||| | | ||||}|s|| zSt|j|j|j|j|j| j| j| jS)N)r.rrrrr r rrrrr) rr_rrrr#r$rr&rr r rrr$decoder_hidden_statesdecoder_attentionsr%encoder_last_hidden_staterencoder_attentions)rr r9 isinstancer rrto_tupler:r rr$rrr%)rTr.rdecoder_input_idsdecoder_attention_maskdecoder_position_idsrdecoder_head_maskr#encoder_outputsr$rdecoder_inputs_embedsr&rr r rrWdecoder_outputss r4r\zTFBlenderbotMainLayer.call`sb.%9$D $++JjJj   ""ll#-#+"3%9'!+ OO=N!O/"1!"4474H14Loa0RV14_1E1I?1-tO Z%G-668O,, 1-"1!"4#1'!5+//!5#' ""_4 4#-??+;;"1"?"?.99,==&5&G&G"1"?"?.99  rGc|jryd|_tj|jjdz|jj zdz5|jj ddddt|ddMtj|jj 5|jj ddddt|ddNtj|jj 5|jj ddddyy#1swYxYw#1swYqxYw#1swYyxYw)NT/r9r:) rr#rr@rAr[rrr9r:rs r4rzTFBlenderbotMainLayer.builds ::  ]]4;;99C?$++BRBRRUXX Y $ KK  d # $ 4D ) 5t||001 ) ""4( ) 4D ) 5t||001 ) ""4( ) ) 6  $ $ ) ) ) )s$D55EE 5D>E  ErNNNNNNNNNNNNNNNNF)rS)Optional[Union[Tuple, TFBaseModelOutput]]rQ) rerfrgrrrSrCrFrr\rrirjs@r4r4r4Dsu#LP 0 #!!EI"!%L CL L \)rGr4zXThe bare BLENDERBOT Model outputting raw hidden-states without any specific head on top.ceZdZd fd ZdZdZed fd Zee e jde e ee d d dZdZdd ZxZS)TFBlenderbotModelcPt||g|i|t|d|_y)Nrr)rRrSr4rrTrinputsrWrXs r4rSzTFBlenderbotModel.__init__s( 3&3F3*6@ rGc.|jjSrQrr9rs r4 get_encoderzTFBlenderbotModel.get_encoderzz!!!rGc.|jjSrQrr:rs r4 get_decoderzTFBlenderbotModel.get_decoderrbrGc|dk(r1ddlm}tjdt|j |St ||g|i|S)Nfacebook/blenderbot-90Mr)TFBlenderbotSmallModelThe checkpoint `facebook/blenderbot-90M` is deprecated. In the future, please use the identical checkpoint `facebook/small_blenderbot-90M` with `TFBlenderbotSmallForConditionalGeneration.from_pretrained('facebook/small_blenderbot-90M')` instead.)blenderbot_smallrhwarningswarn FutureWarningfrom_pretrainedrR)clspretrained_model_name_or_path model_argsrWrhrXs r4rnz!TFBlenderbotModel.from_pretrainedsU (,E E A MM  *99:WX Xw&'D\z\U[\\rGzbatch_size, sequence_length) checkpoint output_typerc J|j||||||||| | | | | ||||}|S)N)r.rrOrPrQrrRr#rSr$rrTr&rr r r)r)rTr.rrOrPrQrrRr#rSr$rrTr&rr r rrWoutputss r4r\zTFBlenderbotModel.callsQ8**)/#9!5/!5++'"7/!5## (rGc |jjr"tj|jdnd}|jj rtj |jnd}|jjrtj |jnd}|jjrtj |jnd}|jj rtj |jnd}|jjrtj |jnd}t|j|||||j||S)NrrH)rr&r#rr$r r&rIrrJr%rrLr rrKrToutputpkvdec_hs dec_attns cross_attnsenc_hs enc_attnss r4serving_outputz TFBlenderbotModel.serving_outputs59[[5J5Jbhhv--.q1PTGK{{GgGg%%f&B&BCmqGK{{GdGdB(()B)BCjn GK{{GdGdb**6+B+BCjn GK{{GgGg%%f&B&BCmqGK{{GdGdB(()B)BCjn #$66"(((&,&F&F"((  rGc|jryd|_t|ddNtj|jj 5|jj ddddyy#1swYyxYw)NTr)rrr#rrr[rrs r4rzTFBlenderbotModel.build&sg ::  4$ ' 3tzz/ '   & ' ' 4 ' 's A11A:rrpz!Optional[Union[str, os.PathLike]]rX)$r.rcrrcrOrcrPrcrQrcrrcrRrcr#rcrSrYr$List[tf.Tensor] | NonerrcrTrcr&rrrr rr rrrrz-Union[Tuple[tf.Tensor], TFSeq2SeqModelOutput]rQ)rerfrgrSrare classmethodrnrrBLENDERBOT_INPUTS_DOCSTRINGformatr_CHECKPOINT_FOR_DOCr _CONFIG_FOR_DOCr\rrrirjs@r4r[r[sa A "" ] ]*+F+M+MNk+lm&($'++/.23715&*.215EI26*.26$(,0/3&*#(%)#))), ) !1 ) / )$),)/)C)0)() 0)")*) -!)"$#)$!%)( 7)) n)X &'rGr[c(eZdZdZfdZdZxZS) BiasLayerz Bias as a layer. It is used for serialization purposes: `keras.Model.save_weights` stores on a per-layer basis, so all weights have to be registered in a layer. c \t|dd|i||j|||||_y)Nr[r[shape initializer trainablers)rRrS add_weightr~)rTrrrr[rWrXs r4rSzBiasLayer.__init__6s3 -d-f-OOU _hOi rGc ||jzSrQ)r~)rTxs r4r\zBiasLayer.call=s499}rGrdrjs@r4rr0s jrGrzRThe BLENDERBOT Model with a language modeling head. Can be used for summarization.cNeZdZddgZfdZdZdZdZdZdZ d Z e dfd Z e eeeee ee d dd Zd Z ddZddZxZS)$TFBlenderbotForConditionalGenerationz!model.encoder.embed_tokens.weightz!model.decoder.embed_tokens.weightct||g|i|t|d|_|j|_t dd|j gdd|_y)Nrrfinal_logits_biasrr>Fr)rRrSr4rr&rr< bias_layerr]s r4rSz-TFBlenderbotForConditionalGeneration.__init__KsW 3&3F3*6@ ))#$Q0A0A,BPWch rGc.|jjSrQrdrs r4rez0TFBlenderbotForConditionalGeneration.get_decoderTrbrGc.|jjSrQr`rs r4raz0TFBlenderbotForConditionalGeneration.get_encoderWrbrGc"|jSrQ)rCrs r4get_output_embeddingsz:TFBlenderbotForConditionalGeneration.get_output_embeddingsZs((**rGc&|j|yrQ)rF)rTvalues r4set_output_embeddingsz:TFBlenderbotForConditionalGeneration.set_output_embeddings]s !!%(rGc2d|jjiS)Nr)rr~rs r4get_biasz-TFBlenderbotForConditionalGeneration.get_bias`s#T__%9%9::rGc|djd}tdd|gdd|_|jjj |dy)Nrrrr>Fr)rrrr~assign)rTrr<s r4set_biasz-TFBlenderbotForConditionalGeneration.set_biascsR./55b9 #$Q O\a  ##E*=$>?rGc|dk(r1ddlm}tjdt|j |St ||g|i|S)Nrgr))TFBlenderbotSmallForConditionalGenerationri)rjrrkrlrmrnrR)rorprqrWrrXs r4rnz4TFBlenderbotForConditionalGeneration.from_pretrainedksU (,E E T MM  =LLMjk kw&'D\z\U[\\rG)rsrc0|tj||jjk(tjtj t |d|j|}d} |7| 5t||jj|jj}|j|||| |||||| | | | ||||}tj|d|jjjd}|j|}|dn|j||}|s|f|ddz}||f|zS|St!|||j"|j$|j&|j(|j*|j,|j. S) a labels (`tf.tensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Returns: Nr F)rrOrSrPrQrrRr#r$rrTr&rr r rrTrr) losslogitsr$rIrJr%rKrrL)r#r(rr/r$r%rr"r5r0rrr@weightsrhf_compute_lossr r$rIrJr%rKrrL)rTr.rrOrPrQrrRr#rSr$rrTr&rr r labelsrru lm_logitsmasked_lm_lossrxs r4r\z)TFBlenderbotForConditionalGeneration.call{sD  XX$++222 6 2D96<<HF I (-B-J$6DKK44dkk6X6X%!** )/+#9!5/!5+'"7/!5## &IIgaj$***;*;*C*CQUV OOI. !'T5I5I&R[5\\GABK/F3A3M^%. YSY Y #33")"?"?&99$55&-&G&G")"?"?&99  rGc |jjr"tj|jdnd}|jj rtj |jnd}|jjrtj |jnd}|jjrtj |jnd}|jj rtj |jnd}|jjrtj |jnd}t|j|||||j||S)Nr)rr$rIrJr%rKrrL)rr&r#rr$r r&rIrrJr%rrLr rrKrws r4rz3TFBlenderbotForConditionalGeneration.serving_outputs59[[5J5Jbhhv--.q1PTGK{{GgGg%%f&B&BCmqGK{{GdGdB(()B)BCjn GK{{GdGdb**6+B+BCjn GK{{GgGg%%f&B&BCmqGK{{GdGdB(()B)BCjn  =="(((&,&F&F"((  rGc | |ddddf}|,tjj|ddddddf} n:||ddjd} n"tj|jd} d| ||||| ||||d S)NrT)r9 exclusiverrr) r.rSr$rOrrPrQrrRr#r&)r#rcumsumrr<) rTrOr$rrPrrRr#r&rSrWrQs r4prepare_inputs_for_generationzBTFBlenderbotForConditionalGeneration.prepare_inputs_for_generations  & 1!RS& 9  ! -#%77>>2Hr]a>#bcdfhfici#j  (#21#5a#8#>#>q#A #%88,=,C,CA,F#G ..!2,&<$8"!2$8"  rGc|jryd|_t|ddMtj|jj 5|jj ddddt|ddNtj|jj 5|jj ddddyy#1swYexYw#1swYyxYw)NTrr)rrr#rrr[rrrs r4rz*TFBlenderbotForConditionalGeneration.build s ::  4$ ' 3tzz/ '   & ' 4t , 8t334 ,%%d+ , , 9 ' ' , ,s C %C CC r)NNNNNNNNNNNNNNNNNF)&r.rcrrcrOrcrPrcrQrcrrcrRrcr#rcrSrYr$rrrcrTrcr&rrrr rr rrrcrrrz*Union[Tuple[tf.Tensor], TFSeq2SeqLMOutput])NNNNNNNNrQ)rerfrg"_keys_to_ignore_on_load_unexpectedrSrerarrrrrrnrrrrr rrBLENDERBOT_GENERATION_EXAMPLEr\rrrrirjs@r4rrAs -,*&  ""+);@ ] ]*+FG+rs" ///SR7   H %8$2;$ 6CU\\-C-CC*gBELL..gBVJJu||11JJ\EJu||11EJP "3 'R!:K\h&%,,,,h&h&Vm&%,,,,m&m&`x)ELL..x)x)v^h'3h' h'X """XN,+FHdN, N,rG