sgdZddlmZddlmZddlmZmZmZm Z ddl Z ddl Z ddlmZddlmZmZmZdd lmZmZmZmZmZmZmZmZmZmZdd lm Z m!Z!m"Z"dd l#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)d d l*m+Z+e(jXe-Z.dZ/dZ0GddejbjdZ3GddejbjdZ4GddejbjdZ5eGddejbjdZ6GddeZ7eGdde$Z8dZ9dZ:e&de9Gdd e7Z;e&d!e9Gd"d#e7eZ<e&d$e9Gd%d&e7Z=e&d'e9Gd(d)e7eZ>y)*zTF 2.0 OpenAI GPT-2 model.) annotations) dataclass)ListOptionalTupleUnionN)get_tf_activation)+TFBaseModelOutputWithPastAndCrossAttentions#TFCausalLMOutputWithCrossAttentions"TFSequenceClassifierOutputWithPast) TFCausalLanguageModelingLossTFConv1DTFModelInputTypeTFPreTrainedModelTFSequenceClassificationLossTFSequenceSummaryget_initializerkeraskeras_serializable unpack_inputs)check_embeddings_within_bounds shape_liststable_softmax) ModelOutputadd_code_sample_docstringsadd_start_docstrings%add_start_docstrings_to_model_forwardloggingreplace_return_docstrings) GPT2Configzopenai-community/gpt2r"c\eZdZd fd ZdZedZd dZdZdZ d dZ d dZ xZ S) TFAttentionc t|di||}||jzdk(sJ|j|_||_||_|j |_||_|j r@t|dz||jd|_ t|||jd|_ n!t|dz||jd|_ t|||jd|_ tjj|j|_tjj|j"|_t'|_||_y) Nrc_attninitializer_rangenameq_attnr c_proj)super__init__n_head split_sizescaleoutput_attentionsis_cross_attentionrr)r'r+r,rlayersDropout attn_pdrop attn_dropout resid_pdrop resid_dropoutset pruned_heads embed_dim)selfnxconfigr2r4kwargsn_state __class__s \/var/www/html/venv/lib/python3.12/site-packages/transformers/models/gpt2/modeling_tf_gpt2.pyr/zTFAttention.__init__?s "6"&!+++mm ! !'!9!9"4  " ""7Q;fF^F^emnDK"7B&BZBZaijDK"7Q;fF^F^emnDKwf>V>V]ef !LL001B1BC"\\11&2D2DEE cyNr-)r>headss rD prune_headszTFAttention.prune_headsXs rEctj|dddf}tj|}|||z |zk\}tj||S)z 1's in the lower triangle, counting from the lower right corner. Same as tf.matrix_band_part(tf.ones([nd, ns]), -1, ns-nd), but doesn't produce garbage on TPUs. N)tfrangecast)ndnsdtypeijms rDcausal_attention_maskz!TFAttention.causal_attention_mask[sJ HHRLD ! HHRL R" wwq%  rEctj||d}|jrOtjt |d|j } |tj j| z }|jsVt |\} } } } |j| | |j } tj| dd| | g} || zdd| z zz }|&tj||j }||z}t|d}|j||}|||z}tj||g}|r|j||S) NT transpose_brPr!g@axistraining)rKmatmulr2rMrrPmathsqrtr4rTreshaperr8append)r>qkvattention_mask head_maskr3r]wdk_rNrOboutputss rD_attnzTFAttention._attnfs0 IIa - ::Ar*!'':BBGGLL$$A&&&a=LAq"b**2r*AA 1q!Rn-AAq1u %A  %WW^177CNN"A 12 &   a(  3  I A99Q?#  NN1 rEctj|gd}t|}|dd|d|dzgz}tj||S)Nrr&r!r rX)rK transposerrar>xx_shape new_x_shapes rD merge_headszTFAttention.merge_headssL LLL )Q-crlgbkGBK&?%@@ zz![))rEct|}|dd|j|d|jzgz}tj||}tj|dS)NrXro)rr0rKrarqrrs rD split_headszTFAttention.split_headssRQ-crldkk72;$++3M%NN JJq+ &||A|,,rEc  |Wt|ds td|j|} |j|} t j | dd\} } |}n-|j|}t j |dd\} } } |j | } |j | } |j | } |Mt j|dd\}}t j|| gd} t j|| gd} |rt j| | gd}nd }|j| | | |||| }|d}|j|}|j|}|j|| }||g|d dz}|S) Nr+zIf class is used as cross attention, the weights `q_attn` have to be defined. Please make sure to instantiate class with `GPT2Attention(..., is_cross_attention=True)`.r&rZr r)r[numrprGr\r!)hasattr ValueErrorr+r'rKsplitrxunstackconcatstackrmrvr,r:)r>rs layer_pastrfrgencoder_hidden_statesencoder_attention_mask use_cacher3r]querykv_outkeyvaluepast_key past_valuepresent attn_outputsarls rDcallzTFAttention.calls ! ,4* p KKNE[[!67F&!!4JC3N AA "AA 6 E3  's#  '  !#%::jqa#H Hj))XsO"5CIIz51;E hhU|!4GGzz%e^YPaltzu O   Q  KKN   q8  4g,ab!11rEc"|jryd|_|jrd|jz}nd|jz}t|ddZt j |j j5|j jdd|jgdddt|ddPt j |jj5|jjdd|gdddt|dd[t j |jj5|jjdd|jgdddyy#1swYxYw#1swY~xYw#1swYyxYw)NTr&r r,r'r+) builtr4r=getattrrK name_scoper,r*buildr'r+)r> input_shape c_attn_shapes rDrzTFAttention.buildsL ::   " "t~~-Lt~~-L 44 ( 4t{{//0 @ !!4t~~">? @ 44 ( 4t{{//0 > !!4|"<= > 44 ( 4t{{//0 @ !!4t~~">? @ @ 5  @ @ > > @ @s$6)E-E9:)F-E69FF)FFFrG) __name__ __module__ __qualname__r/rI staticmethodrTrmrvrxrr __classcell__rCs@rDr$r$>sB!2 !!B* - 1f@rEr$c.eZdZfdZddZddZxZS)TFMLPc `t|di||j}t|||jd|_t|||jd|_t|j|_ tjj|j|_||_||_y)Nc_fcr(r,r-)r.r/n_embdrr)rr,r activation_functionactrr5r6r9dropoutintermediate_sizer=)r>rBr@rAr?rCs rDr/zTFMLP.__init__s "6" ]]WbFV>V]ef $V%?%?@||++F,>,>? !(rEc|j|j|}|j|}|j||}|S)Nr\)rrr,r)r>rsr]hh2s rDrz TFMLP.calls< HHTYYq\ " [[^ \\"x\ 0 rEc|jryd|_t|ddZtj|jj 5|jj dd|jgdddt|dd[tj|jj 5|jj dd|jgdddyy#1swYrxYw#1swYyxYw)NTrr,) rrrKrrr*rrr,r=r>rs rDrz TFMLP.builds ::  4 & 2tyy~~. F tT-C-C DE F 44 ( 4t{{//0 @ !!4t~~">? @ @ 5 F F @ @s )C%2)C1%C.1C:rrGrrrr/rrrrs@rDrrs @rErc2eZdZdfd Z ddZddZxZS)TFBlockc Ht|d i||j}|j |jnd|z}tj j |jd|_t|||d|_ tj j |jd|_ |jrEt|||dd |_ tj j |jd |_t||d |_|j"|_y) Nln_1epsilonr*attnr*ln_2crossattentionT)r*r4 ln_cross_attnmlpr-)r.r/rn_innerrr5LayerNormalizationlayer_norm_epsilonrr$rradd_cross_attentionrrrr hidden_size)r>r@r2rAr? inner_dimrCs rDr/zTFBlock.__init__s "6" ]]&,nn&@FNNa"f LL33Frsrrfrgrrrr3r]r output_attnrlcaoutput_cross_attnrSs rDrz TFBlock.calls4 IIaLii !)"&#'/   Nab/ E ! ,4!12 =dVDZZ ##A&B $ 3 3-#&;'="3!!4 ! #1%BBA 1!" 55G IIaL HHQH * E#-rEcJ|jryd|_t|ddZtj|jj 5|jj dd|jgdddt|ddMtj|jj 5|jj ddddt|ddZtj|jj 5|jj dd|jgdddt|ddMtj|jj 5|jj ddddt|ddMtj|jj 5|jj ddddt|dd[tj|jj 5|jj dd|jgdddyy#1swYxYw#1swYxYw#1swYAxYw#1swYxYw#1swYxYw#1swYyxYw)NTrrrrrr) rrrKrrr*rrrrrrrrs rDrz TFBlock.buildDs ::  4 & 2tyy~~. @ tT-=-= >? @ 4 & 2tyy~~. & % & 4 & 2tyy~~. @ tT-=-= >? @ 4 % 1txx}}- %t$ % 4)4 0 <t22778 0##))$/ 0 4$ / ;t11667 I""(($d6F6F)GH I I < @ @ & & @ @ % % 0 0 I IsH )I2I' )I43J J ')JI$'I14I>J  JJ"rrGrrs@rDrrs.6:xIrErceZdZeZfdZdZdZdZe d ddZ d dZ xZ S) TFGPT2MainLayerc t||i|||_|j|_|j|_|j |_|j |_|j|_ |j|_ |j|_ |j|_ tjj|j |j"t%|jd|_tjj|j|jt%|jd|_tjj+|j,|_t1|jDcgc]}t3|dd|c}|_tjj7|j8d|_|j"|_ycc}w) Nwte) input_dim output_dimembeddings_initializerr*wpeTzh_._)r2r*ln_fr)r.r/r@r3output_hidden_statesruse_return_dict return_dictn_layernum_hidden_layersr n_positionsr)rr5 Embedding vocab_sizerrrrr6 embd_pdropdroprLrrrrrr=)r>r@inputsrArQrCs rDr/zTFGPT2MainLayer.__init__`s &+F+ !'!9!9$*$?$?!))!11!'mm !--!'!9!9<<))''))#263K3K#L *  <<))((}}#263K3K#L *  LL(():):; HMfnnH]^1'&T!:>^LL33Fs rDget_input_embeddingsz$TFGPT2MainLayer.get_input_embeddingss xxrEc||_yrGr)r>new_embeddingss rDset_input_embeddingsz$TFGPT2MainLayer.set_input_embeddingss !rEct)zv Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} )NotImplementedError)r>heads_to_prunes rD _prune_headszTFGPT2MainLayer._prune_headss "!rEc "| | td|'t|}tj|d|dg}n|t|dd}n td|d}dgt |j z}nt|ddd}|1tj tj||d|zd}|t|}tj||ddd|df}tjd}tj||j }tjtj||tjd }|jjrf| dtj| |j } t t| }|d k(r| dddddddf}|d k(r | ddddddf}dz d z}nd}|} |tdg|j z}tj|dt|dg}|1t#||jj$|j'|}|j)|}|6tj|dt|dg}|j'|}ntjd }tj||j }tj||j }||z|z}|j+||}|t|dgz}| rdnd}| rdnd}| r|jjrdnd}| rdnd}t-t/|j |D]w\}\}}| r|tj||fz}|||||||| | | | } | dd \}}!| r||!fz}| sL|| d fz}|jjsl|o|| d fz}y|j1|}tj||}| r||fz}| r/|dddgzt|dddz"t3"fd|D}| st3d|||||fDSt5|||||S)NzDYou cannot specify both input_ids and inputs_embeds at the same timerXz5You have to specify either input_ids or inputs_embedsrrprZr!g?rYgr r&gr\r-c3JK|]}tj|ywrG)rKra).0tattention_output_shapes rD z'TFGPT2MainLayer.call.. s"aQ2::a1G#H"as #c3$K|]}|| ywrGr-)rres rDrz'TFGPT2MainLayer.call..#s=s)last_hidden_statepast_key_values hidden_states attentionscross_attentions)r|rrKralenr expand_dimsrLconstantrMrPmultiplysubtractr@rrrrrrrr enumerateziprtupler )#r> input_idsrrftoken_type_ids position_idsrg inputs_embedsrrrr3rrr]r past_lengthattention_mask_shapeone_cstnum_dims_encoder_attention_maskencoder_extended_attention_maskposition_embedstoken_type_embedsr output_shapepresentsall_attentionsall_cross_attentionsall_hidden_statesrQblockrrlrrs# @rDrzTFGPT2MainLayer.calls$  ]%>cd d  "$Y/K 9r;r?.CDI  &$]3CR8KTU U  "K#fs466{2O$_Q%7%:;B?K  >>"((; BR]@]*^efgL  % $.n#= ZZ9Ma9PRSUVXlmnXo8pqNkk#&GWW^7==IN[[Wn)Mr{{[cOdeN ;; * */E/Q&(WW-CK`KfKf%g ".1*=S2T.U +.!32HDRSUV2W/.!32HDRVXYIY2Z/035T/TX`.` +.2 +!@  % %!7!77Izz,Z 5Mb5Q0RS  *9dkk6L6L M HHY/M((<0  %ZZZ=WXZ=[8\]N $ 8  " C 0 ''/9L9LMGG$5]=P=PQ%7:KK  -( C "j&?&C%DD "20d%64;;;Z;Zr`d"6BD&/DFFO0L&M P "A"z#$5MS_9`8b$b!! %&!! G&-Ra[ "M7#wj0 !/71:-!?;;227L7X+?71:-+O(/ P2 -0  =,?  1]4D D  %0"%5%rlayers rDrzTFGPT2MainLayer.build1sS ::  4 % 1txx}}- %t$ % 4 % 1txx}}- %t$ % 4 & 2tyy~~. > tT^^ <= > 4d # / &]]5::.&KK%&& & 0 % % % % > >&&s0 F %F?)F"+F. FF"F+.F7 NNNNNNNNNNNNNFrTFModelInputType | Noner4Optional[Tuple[Tuple[Union[np.ndarray, tf.Tensor]]]]rfnp.ndarray | tf.Tensor | NonerrrrrgrrrrrrrrOptional[bool]r3rrrrrr]rreturnzDUnion[TFBaseModelOutputWithPastAndCrossAttentions, Tuple[tf.Tensor]]rG) rrrr" config_classr/rrrrrrrrs@rDrr\s L,>"" .2PT8<8<6:377;?C@D$(,0/3&*#(c *c Nc 6 c 6 c 4 c 1c 5c  =c !>c "c *c -c $c !c N!c c J&rErc0eZdZdZeZdZddgZedZ y)TFGPT2PreTrainedModelz An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained models. transformerzh.\d+.attn.biaszh.\d+.crossattention.biasctjdtjdtjdtjddS)NNNrrrf)rrfrK TensorSpecint32rs rDinput_signaturez%TFGPT2PreTrainedModel.input_signatureOs7 |RXXKP mmL"((IYZ  rEN) rrr__doc__r"rbase_model_prefix"_keys_to_ignore_on_load_unexpectedpropertyrr-rErDrrDs2 L%*<>Z)[&   rErcXeZdZUdZdZded<dZded<dZded<dZded <dZ ded <y) TFGPT2DoubleHeadsModelOutputa( Base class for outputs of models predicting if two sentences are consecutive or not. Args: logits (`tf.Tensor` of shape `(batch_size, num_choices, sequence_length, config.vocab_size)`): Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). mc_logits (`tf.Tensor` of shape `(batch_size, num_choices)`): Prediction scores of the multiple choice classification head (scores for each choice before SoftMax). past_key_values (`List[tf.Tensor]`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): List of `tf.Tensor` of length `config.n_layers`, with each tensor of shape `(2, batch_size, num_heads, sequence_length, embed_size_per_head)`). Contains pre-computed hidden-states (key and values in the attention blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. hidden_states (`tuple(tf.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`. Hidden-states of the model at the output of each layer plus the initial embedding outputs. attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`): Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`. Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. Nz tf.Tensorlogits mc_logitszList[tf.Tensor] | NonerzTuple[tf.Tensor] | Nonerr) rrrrr!__annotations__r"rrrr-rErDr r Zs>6FIIy.2O+2-1M*1*.J'.rEr az This model inherits from [`TFPreTrainedModel`]. Check the superclass documentation for the generic methods the library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads etc.) This model is also a [keras.Model](https://www.tensorflow.org/api_docs/python/tf/keras/Model) subclass. Use it as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage and behavior. TensorFlow models and layers in `transformers` accept two formats as input: - having all inputs as keyword arguments (like PyTorch models), or - having all inputs as a list, tuple or dict in the first positional argument. The reason the second format is supported is that Keras methods prefer this format when passing inputs to models and layers. Because of this support, when using methods like `model.fit()` things should "just work" for you - just pass your inputs and labels in any format that `model.fit()` supports! If, however, you want to use the second format outside of Keras methods like `fit()` and `predict()`, such as when creating your own layers or models with the Keras `Functional` API, there are three possibilities you can use to gather all the input Tensors in the first positional argument: - a single Tensor with `input_ids` only and nothing else: `model(input_ids)` - a list of varying length with one or several input Tensors IN THE ORDER given in the docstring: `model([input_ids, attention_mask])` or `model([input_ids, attention_mask, token_type_ids])` - a dictionary with one or several input Tensors associated to the input names given in the docstring: `model({"input_ids": input_ids, "token_type_ids": token_type_ids})` Note that when creating models and layers with [subclassing](https://keras.io/guides/making_new_layers_and_models_via_subclassing/) then you don't need to worry about any of this, as you can just pass inputs like you would to any other Python function! Parameters: config ([`GPT2Config`]): Model configuration class with all the parameters of the model. Initializing with a config file does not load the weights associated with the model, only the configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. a Args: input_ids (`Numpy array` or `tf.Tensor` of shape `(batch_size, input_ids_length)`): `input_ids_length` = `sequence_length` if `past_key_values` is `None` else `past_key_values[0].shape[-2]` (`sequence_length` of input past key value states). Indices of input sequence tokens in the vocabulary. If `past_key_values` is used, only input IDs that do not have their past calculated should be passed as `input_ids`. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.__call__`] and [`PreTrainedTokenizer.encode`] for details. [What are input IDs?](../glossary#input-ids) past_key_values (`List[tf.Tensor]` of length `config.n_layers`): Contains pre-computed hidden-states (key and values in the attention blocks) as computed by the model (see `past_key_values` output below). Can be used to speed up sequential decoding. The token ids which have their past given to this model should not be passed as input ids as they have already been computed. attention_mask (`tf.Tensor` or `Numpy array` of shape `(batch_size, sequence_length)`, *optional*): Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. If `past_key_values` is used, `attention_mask` needs to contain the masking strategy that was used for `past_key_values`. In other words, the `attention_mask` always has to have the length: `len(past_key_values) + len(input_ids)` [What are attention masks?](../glossary#attention-mask) token_type_ids (`tf.Tensor` or `Numpy array` of shape `(batch_size, sequence_length)`, *optional*): Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`: - 0 corresponds to a *sentence A* token, - 1 corresponds to a *sentence B* token. [What are token type IDs?](../glossary#token-type-ids) position_ids (`tf.Tensor` or `Numpy array` of shape `(batch_size, sequence_length)`, *optional*): Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`. [What are position IDs?](../glossary#position-ids) head_mask (`Numpy array` or `tf.Tensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - 1 indicates the head is **not masked**, - 0 indicates the head is **masked**. inputs_embeds (`tf.Tensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert `input_ids` indices into associated vectors than the model's internal embedding lookup matrix. output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. This argument can be used only in eager mode, in graph mode the value in the config will be used instead. output_hidden_states (`bool`, *optional*): Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for more detail. This argument can be used only in eager mode, in graph mode the value in the config will be used instead. return_dict (`bool`, *optional*): Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. This argument can be used in eager mode, in graph mode the value will always be set to True. training (`bool`, *optional*, defaults to `False`): Whether or not to use the model in training mode (some modules like dropout modules have different behaviors between training and evaluation). z^The bare GPT2 Model transformer outputting raw hidden-states without any specific head on top.ceZdZfdZeeeeee e  d ddZ ddZ xZ S) TFGPT2ModelcPt||g|i|t|d|_yNrrr.r/rrr>r@rrArCs rDr/zTFGPT2Model.__init__) 3&3F3*6 FrE checkpoint output_typercD|j||||||||| | | | | |}|S)a encoder_hidden_states (`tf.Tensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if the model is configured as a decoder. encoder_attention_mask (`tf.Tensor` of shape `(batch_size, sequence_length)`, *optional*): Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in the cross-attention if the model is configured as a decoder. Mask values selected in `[0, 1]`: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. past_key_values (`Tuple[Tuple[tf.Tensor]]` of length `config.n_layers`) contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding. If `past` are used, the user can optionally input only the last `decoder_input_ids` (those that don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `decoder_input_ids` of shape `(batch_size, sequence_length)`. use_cache (`bool`, *optional*, defaults to `True`): If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see `past`). Set to `False` during training, `True` during generation rrrfrrrgrrrrr3rrr])r)r>rrrfrrrgrrrrr3rrr]rls rDrzTFGPT2Model.callsKZ""+))%'"7#9/!5## "rEc|jryd|_t|ddNtj|jj 5|jj ddddyy#1swYyxYwNTrrrrKrrr*rrs rDrzTFGPT2Model.build5m ::  4 - 9t//445 -  &&t, - - : - - A11A:r r rG)rrrr/rrGPT2_INPUTS_DOCSTRINGr_CHECKPOINT_FOR_DOCr _CONFIG_FOR_DOCrrrrs@rDr%r%s  G*+@A&?$.2PT8<8<6:377;?C@D$(,0/3&*#(7*7N76 7 6 7 4 71757 =7!>7"7*7-7$7!7 N!7 B7r-rEr%z The GPT2 Model transformer with a language modeling head on top (linear layer with weights tied to the input embeddings). ceZdZfdZdZdZddZeee e e e e  d d dZd dZxZS) TFGPT2LMHeadModelcPt||g|i|t|d|_yr'r(r)s rDr/zTFGPT2LMHeadModel.__init__Fr*rEc"|jSrG)rrs rDget_output_embeddingsz'TFGPT2LMHeadModel.get_output_embeddingsJs((**rEc&|j|yrG)r)r>rs rDset_output_embeddingsz'TFGPT2LMHeadModel.set_output_embeddingsMs !!%(rEc |jdd}|rrrrrArrrfs rDprepare_inputs_for_generationz/TFGPT2LMHeadModel.prepare_inputs_for_generationPs$4d; ^^F1b5M26F)!#q"u0Er!Jzz.$7 $4d;  %,*>77>>.rT>RL!~~l1b5.A2F  ,(.",   rEr+c|j||||||||| | | | | |}|d}tj||jjjd}d}|(|ddddf}|ddddf}|j ||}| s|f|ddz}||f|zS|St |||j|j|j|jS) a encoder_hidden_states (`tf.Tensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if the model is configured as a decoder. encoder_attention_mask (`tf.Tensor` of shape `(batch_size, sequence_length)`, *optional*): Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in the cross-attention if the model is configured as a decoder. Mask values selected in `[0, 1]`: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. past_key_values (`Tuple[Tuple[tf.Tensor]]` of length `config.n_layers`) contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding. If `past` are used, the user can optionally input only the last `decoder_input_ids` (those that don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `decoder_input_ids` of shape `(batch_size, sequence_length)`. use_cache (`bool`, *optional*, defaults to `True`): If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see `past`). Set to `False` during training, `True` during generation labels (`tf.Tensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the cross entropy classification loss. Indices should be in `[0, ..., config.vocab_size - 1]`. r/rTrVNrXr!)lossr!rrrr) rrKr^rweightshf_compute_lossr rrrr)r>rrrfrrrgrrrrr3rrlabelsr]transformer_outputsrr!rEshifted_logitsoutputs rDrzTFGPT2LMHeadModel.callisb#..+))%'"7#9/!5#/  ,A. =$*:*:*>*>*F*FTXY  #AssF^NAqrE]F''?DY!4QR!88F)-)9TGf$ Ev E2/??-;;*550AA   rEc|jryd|_t|ddNtj|jj 5|jj ddddyy#1swYyxYwr1r2rs rDrzTFGPT2LMHeadModel.buildr3r4r)NNNNNNNNNNNNNNF) rr rr rfrrrrrrgrrrrrrrrrr3rrrrrrHrr]rrzrCrrr5rr6r r7rrrrs@rDr9r9>s8G+) 2*+@A&7$.2PT8<8<6:377;?C@D$(,0/3&*04#(!O *O NO 6 O 6 O 4 O 1O 5O  =O !>O "O *O -O $O .O !!O " F#O BO b-rEr9a{ The GPT2 Model transformer with a language modeling and a multiple-choice classification head on top e.g. for RocStories/SWAG tasks. The two heads are two linear layers. The language modeling head has its weights tied to the input embeddings, the classification head takes as input the input of a specified classification token index in the input sequence). ceZdZfdZeeeeee  d ddZ e dZ ddZ xZS) TFGPT2DoubleHeadsModelct||g|i|d|_t|d|_t ||j d|_y)Nr!rrmultiple_choice_headr()r.r/ num_labelsrrrr)rPr)s rDr/zTFGPT2DoubleHeadsModel.__init__sL 3&3F3*6 F$5 f&>&>E[% !rE)r-rc | t|}nt|dd}|d}|tj|d|fnd}|tj|d|fnd}|tj|d|fnd}|tj|d|fnd}|j|||||||dd| | | | | }|d}tj||t|ddz}| r| r|jdd|fz}nd}tj ||jj jd}|j||| }tj|d}| s ||f|d dzSt|||j||j S) a mc_token_ids (`tf.Tensor` or `Numpy array` of shape `(batch_size, num_choices)`, *optional*, default to index of the last token of the input): Index of the classification token in each input sequence. Selected in the range `[0, input_ids.size(-1) - 1]`. Return: Examples: ```python >>> import tensorflow as tf >>> from transformers import AutoTokenizer, TFGPT2DoubleHeadsModel >>> tokenizer = AutoTokenizer.from_pretrained("openai-community/gpt2") >>> model = TFGPT2DoubleHeadsModel.from_pretrained("openai-community/gpt2") >>> # Add a [CLS] to the vocabulary (we should train it also!) >>> num_added_tokens = tokenizer.add_special_tokens({"cls_token": "[CLS]"}) >>> embedding_layer = model.resize_token_embeddings( ... len(tokenizer) ... ) # Update the model embeddings with the new vocabulary size >>> choices = ["Hello, my dog is cute [CLS]", "Hello, my cat is cute [CLS]"] >>> encoded_choices = [tokenizer.encode(s) for s in choices] >>> cls_token_location = [tokens.index(tokenizer.cls_token_id) for tokens in encoded_choices] >>> input_ids = tf.constant(encoded_choices)[None, :] # Batch size: 1, number of choices: 2 >>> mc_token_ids = tf.constant([cls_token_location]) # Batch size: 1 >>> outputs = model(input_ids, mc_token_ids=mc_token_ids) >>> lm_prediction_scores, mc_prediction_scores = outputs[:2] ```NrXr/rTrVr\rZr!)r!r"rrr) rrKrarrr^rrFrPsqueezer rr)r>rrrfrrrgr mc_token_idsrr3rrr] input_shapes seq_lengthflat_input_idsflat_attention_maskflat_token_type_idsflat_position_idsrIrr lm_logitsr"s rDrzTFGPT2DoubleHeadsModel.callsj  %i0L%m4Sb9L!"% DMDYIJ/?@_cN\Nhbjj"j9IJnrN\Nhbjj"j9IJnrJVJbBJJ|b*5EFhl"..$+..*'"&#'/!5#/  ,A.  =,MAZ[][^A_2_` /!4 A A#2 F-IY Y  $ IImT-=-=-A-A-I-IW[\ --m\T\-] JJyr2 y),?,CC C+/??+*55   rEctjdtjdtjdtjdtjdtjddS)N)NNNrrrfrrT)rrfrTrrs rDrz&TFGPT2DoubleHeadsModel.input_signatureBsM'9288+V mm,>O_`MM,~V  rEc|jryd|_t|ddMtj|jj 5|jj ddddt|ddNtj|jj 5|jj ddddyy#1swYexYw#1swYyxYw)NTrrP)rrrKrrr*rrPrs rDrzTFGPT2DoubleHeadsModel.buildJs ::  4 - 9t//445 -  &&t, - 4/ 6 Bt88==> 6))//5 6 6 C - - 6 6s C %C CC  NNNNNNNNNNNNF)rr rr rfrrrrrrgrrrrTrrrr3rrrrrr]rrz5Union[TFGPT2DoubleHeadsModelOutput, Tuple[tf.Tensor]]rG)rrrr/rrr5r r r7rrrrrrs@rDrNrNs *+@A+GVef.2PT8<8<6:377;6:$(,0/3&*#(a *a Na 6 a 6 a 4 a 1a 5a 4a "a *a -a $a !a  ?a gBa F   6rErNa The GPT2 Model transformer with a sequence classification head on top (linear layer). [`TFGPT2ForSequenceClassification`] uses the last token in order to do the classification, as other causal models (e.g. GPT-1) do. Since it does classification on the last token, it requires to know the position of the last token. If a `pad_token_id` is defined in the configuration, it finds the last token that is not a padding token in each row. If no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in each row of the batch). ceZdZfdZeeeedee  d ddZ ddZ xZ S) TFGPT2ForSequenceClassificationc t||g|i||j|_tjj |jt |jdd|_t|d|_ ||_ y)NscoreF)kernel_initializerr*use_biasrr) r.r/rQrr5Denserr)rbrrr@r)s rDr/z(TFGPT2ForSequenceClassification.__init__fsw 3&3F3 ++\\''   .v/G/GH (  +6 F rEzmicrosoft/DialogRPT-updownr+c$|j||||||||| | | |  }|d}|j|}t|}d}|jjd}n|t j t jt jj||jj|jddz }t j|dk\||jddz }t j||dd}n.d}tj|j j"dd}| |jj|ddk(sJd t j$|s |d|d|f}|j't j(| dgt j(|d|j*g}||n|}| s|f|ddz}||f|zS|St-|||j.|j0|j2 S) z labels (`tf.Tensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the cross entropy classification loss. Indices should be in `[0, ..., config.vocab_size - 1]`. ) rrrfrrrgrrr3rrr]rNrXrZr!) batch_dimsr[z will not detect padding tokens in `inputs_embeds`. Results may be unexpected if using padding tokens in conjunction with `inputs_embeds.`z=Cannot handle batch sizes > 1 if no padding token is defined.)rEr!rrr)rrbrr@ pad_token_idrKargmaxrMr_equalrPwhereshapegatherlogger warning_oncerCr is_tensorrGrarQr rrr)r>rrrfrrrgrrr3rrrHr]rIrr! logits_shape in_logitssequence_lengthsrE pooled_logitsrKs rDrz$TFGPT2ForSequenceClassification.callrsA8#..+))%'/!5#/  ,A. M*!&)  ;; # # +! $IIbggbggmmIt{{?W?W&XZcZiZijqst!$&88,<,ACSU^UdUdegUhklUl#m IIf.>1STU #% ##~~../0^^   ((4 Q18L ON OL<< 01"1|A#68H#HI '' 6B4(@"**YY[]a]l]lXmBnoD%.%:  #%(;AB(??F)-)9TGf$ Ev E1 /??-;;*55   rEc|jryd|_t|dddtj|jj 5|jj dd|jjgdddt|ddNtj|jj 5|jj ddddyy#1swYexYw#1swYyxYw)NTrbr) rrrKrrbr*rr@rrrs rDrz%TFGPT2ForSequenceClassification.builds ::  4$ ' 3tzz/ C   $dkk.@.@!AB C 4 - 9t//445 -  &&t, - - : C C - -s 3C"<C."C+.C7r^)rr rr rfrrrrrrgrrrrrr3rrrrrrHrr]rrz;Union[TFSequenceClassifierOutputWithPast, Tuple[tf.Tensor]]rG) rrrr/rrr5rr r7rrrrs@rDr`r`Vs  *+@A/6$.2PT8<8<6:377;$(,0/3&*04#(O *O NO 6 O 6 O 4 O 1O 5O "O *O -O $O .O !O  EO BO b -rEr`)?r __future__r dataclassesrtypingrrrrnumpynp tensorflowrKactivations_tfr modeling_tf_outputsr r r modeling_tf_utilsrrrrrrrrrrtf_utilsrrrutilsrrrrrr configuration_gpt2r" get_loggerrrnr6r7r5Layerr$rrrrr GPT2_START_DOCSTRINGr5r%r9rNr`r-rErDrs !"!///   SR+   H %-X@%,,$$X@v@ELL  @:cIell  cILd&ell((d&d&N - ,  /; /  /F(TAHdK-'K- K-\ B--/KB-B-J @62@6@6F n-&;=Yn-n-rE