sg۰dZddlmZddlZddlZddlmZmZmZm Z ddl Z ddl Z ddlmZddlmZmZmZmZddlmZmZdd lmZmZmZmZmZmZmZm Z dd l!m"Z"m#Z#m$Z$dd l%m&Z&d d l'm(Z(e&jRe*Z+dZ,dZ-dZ.d&dZ/ d'dZ0 d(dZ1d)d*dZ2d+d,dZ3GddejhjjZ6GddejhjjZ7eGddejhjjZ8GddeZ9dZ:dZ;ed e:Gd!d"e9Z<ed#e:Gd$d%e9eZ=y)-zTF 2.0 XGLM model.) annotationsN)AnyOptionalTupleUnion)get_tf_activation)add_code_sample_docstringsadd_start_docstrings%add_start_docstrings_to_model_forwardreplace_return_docstrings)+TFBaseModelOutputWithPastAndCrossAttentions#TFCausalLMOutputWithCrossAttentions)TFCausalLanguageModelingLossTFModelInputTypeTFPreTrainedModelTFSharedEmbeddingsget_initializerkeraskeras_serializable unpack_inputs)check_embeddings_within_bounds shape_liststable_softmax)logging) XGLMConfigzfacebook/xglm-564Mrgחc |dz}tjd|dz z }tjtj|tj | z}tj tj|tj dtj |dz}tjtjtj|tj|gd|df}|dzdk(r.tj|tj|dfgd}|tjtj|t|dftjdt|dftjt|d|z dz t|dfgd}||z}tj|d S) Ni'rdtypeaxisrembed_positionsname)mathlogtfexprangefloat32 expand_dimsreshapeconcatsincoszerosonesrconstant) num_positions embedding_dim padding_idxhalf_dimemb _padding_masks \/var/www/html/venv/lib/python3.12/site-packages/transformers/models/xglm/modeling_tf_xglm.pycreate_sinusoidal_positionsr=<sz!H ((5/X\ *C &&("**5< =C ..-rzzB Kbnn]`ghNi iC **RYYs RVVC[9B]TVDW XCqAiibhh q'9:;!D joa&89:!Z_Q/01C+k9A=z#q?QRS     } ;;s!2 33ctj||k7dd}tjtj|d|j|z|z}tj|tj |zS)z Replace non-padding symbols with their position numbers. Position numbers begin at padding_idx+1. Padding symbols are ignored. This is modified from fairseq's `utils.make_positions`. rrr"r )r*wherecastcumsumr!int64) input_idspast_key_values_lengthr8maskincremental_indicess r<#_create_position_ids_from_input_idsrHSsb 88I,a 3D77299T#:$**MPffjnn 77&bhh 7+ EEr>ct|dd}|d}tj|dz||zdztj}tjtj |d||zS)z Args: We are provided embeddings directly. We cannot infer which are padded so just generate sequential position ids. inputs_embeds: tf.Tensor Returns: tf.Tensor Nr$rr rr")rr*r,rC broadcast_tor.) inputs_embedsrEr8 input_shapesequence_length position_idss r<'_create_position_ids_from_inputs_embedsrO`sg]+CR0K!!nO88K!O_{-JQ-NVXV^V^_L ??2>>,Q? MPf ffr>c |d}|d}tj||ftz}tjt |d}tj |tj |dzt |ddfkd|}|dkDr.tjtj||f|gd}tj|ddddddf|dddfS)zB Make causal mask used for bi-directional self-attention. rrr$r"N) r*r4LARGE_NEGATIVEr,rr@r/r0r3tile)input_ids_shaperEbsztgt_lenrF mask_conds r<_make_causal_maskrXrs ! Ca G 77GW% & 7DD)"-.I 88I 9q=:d;KB;OQR:S TTVY[_ `D!yy"((G-C#DEtLSUV 774dAq()CAq> ::r>ct|d}||n|}tjd}tj||j}tj |ddddddfdd|df}||z t zS)z_ Expands attention_mask from `[bsz, seq_len]` to `[bsz, 1, tgt_seq_len, src_seq_len]`. rN?r )rr*r5rAr!rSrR)rFrVsrc_lenone_cst expanded_masks r< _expand_maskr^sxq!G ,g'Gkk#G 774w}} -DGGDD$!12Q7A4FGM m #~ 55r>c|eZdZdZ d dfd ZddZ d d dZd dZxZS) TFXGLMAttentionz6Multi-headed attention from "Attention Is All You Needc zt|d i|||_||_tj j ||_||z|_|j|z|jk7rtd|jd|d|jdz|_ ||_ tj j||d|_ tj j||d|_tj j||d|_tj j||d |_y) Nz;embed_dim must be divisible by num_heads (got `embed_dim`: z and `num_heads`: z).gk_proj)use_biasr'q_projv_projout_proj)super__init__ embed_dim num_headsrlayersDropoutdropouthead_dim ValueErrorscaling is_decoderDenserbrdrerf)selfrjrkrnrrbiaskwargs __class__s r<rizTFXGLMAttention.__init__s "6"""||++G4 !Y. MMI %$.. 8MdnnM]$YKr3 }}d* $ll((T(Q ll((T(Q ll((T(Q  **9t**U r>c tjtj||||j|jfdS)Nrrrr)r* transposer/rkro)rttensorseq_lenrUs r<_shapezTFXGLMAttention._shapes0||BJJvWdnndmm/\]_kllr>c  |du}t|\}} } |j||jz} |r| |d} |d} n |rE|j|j |d|} |j|j |d|} n|}|j|j |d|} |j|j |d|} t j|d| gd} t j|d| gd} nD|j|j |d|} |j|j |d|} |jr| | f}||jzd|jf}t j|j| | ||} t j| |} t j| |} t| d}t j| | d}t jjt|||jz| |gd ||jz| |fd t| |t jjt||d| |gd |d| |fd t| t j||j }t j|||j| |f|z}t j|||jz| |f}t#|d}|t jjt||jgd|jd t| t j|dt j|||j| |fz}t j|||jz| |f}|j%||}t j|| }t jjt|||jz| |jgd||j| |jfd t| t j&t j|||j| |jfd}t j||| | f}|j)|}t j|||j| |f}|||fS)z#Input shape: Batch x Time x ChannelNrrr$rr"T) transpose_bz$Attention weights should be of size z , but is messagez!Attention mask should be of size r z/Head mask for a single layer should be of size )rr$rrtrainingz `attn_output` should be of size ry)rrdrqr}rbrer*r0rrrkror/matmul debugging assert_equalrAr!rrnrzrf)rt hidden_stateskey_value_statespast_key_valueattention_masklayer_head_maskris_cross_attentionrUrVrj query_states key_states value_states proj_shaper[ attn_weights attn_probs attn_outputs r<callzTFXGLMAttention.calls.T9",]";Wi{{=1DLL@ ."<'*J)!,L T[[1A%BBLJ;;t{{3C'Db#NL  'T[[%?SIJ;;t{{='A2sKLN1$5z#BKJ99nQ&7%FQOLT[[%?SIJ;;t{{='A2sKL ??),7NDNN*B > zz$++lGS"I:V ZZ J7 zz, ; Z(+yyztL  !! | $ 4>> !7G 46dnn8LgW^7_6`a|,-/ "   % LL % %>*a'*7a'8R7ST">235 &  WW^<;M;MNN::lS$..'SZ4[\_mmL::lS4>>5I7T[4\]L%l<  & LL % %?+ Et~~EWX"?346 & ::o}E sDNNGWEIL::lS4>>5I7T[4\]L\\,\B ii L9  !! { # 4>> !7DMM :2CRVR_R_3`2ab{+,. " ll JJ{S$..'4==$Q RT` jjsGY.GH mmK0 "$**\CQXZa;b"c L.88r>c|jryd|_t|ddZtj|jj 5|jj dd|jgdddt|ddZtj|jj 5|jj dd|jgdddt|ddZtj|jj 5|jj dd|jgdddt|dd[tj|jj 5|jj dd|jgdddyy#1swYAxYw#1swYxYw#1swYxYw#1swYyxYw)NTrbrdrerf) builtgetattrr* name_scoperbr'buildrjrdrerfrtrLs r<rzTFXGLMAttention.build*s ::  44 ( 4t{{//0 @ !!4t~~">? @ 44 ( 4t{{//0 @ !!4t~~">? @ 44 ( 4t{{//0 @ !!4t~~">? @ 4T * 6t}}112 B ##T4$@A B B 7 @ @ @ @ @ @ B Bs0 )F32)G)G )G3F=G  GG!)rQFT) rjintrkrrnfloatrrboolrur)r{ tf.Tensorr|rrUr)NNNNF)rrrtf.Tensor | NonerzTuple[Tuple[tf.Tensor]] | NonerrrrrOptional[bool]returnz"Tuple[tf.Tensor, tf.Tensor | None]N) __name__ __module__ __qualname____doc__rir}rr __classcell__rws@r<r`r`s@   VVV V  V  V8m .29=+/,0#(t9 t9+t97 t9 ) t9 * t9!t9 ,t9lBr>r`cbeZdZdfd Z d ddZddZxZS)TFXGLMDecoderLayerc t|d i||j|_t |j|j |j dd|_tjj|j|_ t|j|_tjj|j|_|j rYt |j|j |j dd|_tjj%dd|_tjj%dd|_tjj+|j,d |_tjj+|jd |_tjj%dd |_||_y)NT self_attn)rjrkrnrrr' encoder_attnh㈵>encoder_attn_layer_normepsilonr'self_attn_layer_normfc1r&fc2final_layer_normrg)rhrid_modelrjr`attention_headsattention_dropoutrrrlrmrnr activation_function activation_fnactivation_dropoutadd_cross_attentionrLayerNormalizationrrrsffn_dimrrrconfig)rtrrvrws r<rizTFXGLMDecoderLayer.__init__=sp "6"(nn,,,,  ||++FNN; .v/I/IJ"',,"6"6v7P7P"Q  % % /.. 0000# !D ,1<<+J+J#<,K,D (%*LL$C$CDWm$C$n!<<%%fnn5%A<<%%dnn5%A % ? ?Se ? f r>c 8|} |j|}||ddnd} |j|| ||\}} } |j||}| |z}d} d}|S|} |j|}||ddnd}|j |||||\}}} |j||}| |z}| | z} |} |j |}|j |j|}|j||}|j|}|j||}| |z}|| || fS)a Args: hidden_states (`tf.Tensor`): input to the layer of shape *(batch, seq_len, embed_dim)* attention_mask (`tf.Tensor`): attention mask of size *(batch, 1, tgt_len, src_len)* where padding elements are indicated by very large negative values. encoder_hidden_states (`tf.Tensor`): cross attention input to the layer of shape *(batch, seq_len, embed_dim)* encoder_attention_mask (`tf.Tensor`): encoder attention mask of size *(batch, 1, tgt_len, src_len)* where padding elements are indicated by very large negative values. layer_head_mask (`tf.Tensor`): mask for attention heads in a given layer of size *(decoder_attention_heads,)* cross_attn_layer_head_mask (`tf.Tensor`): mask for heads of the cross-attention module. *(decoder_attention_heads,)* past_key_value (`Tuple(tf.Tensor)`): cached past key and value projection states Nr)rrrrr)rrrrr) rrrnrrrrrrr)rtrrencoder_hidden_statesencoder_attention_maskrcross_attn_layer_head_maskrrresidualself_attn_past_key_valueself_attn_weightspresent_key_valuecross_attn_present_key_valuecross_attn_weightscross_attn_past_key_values r<rzTFXGLMDecoderLayer.call^s4!11-@ :H9S>"1#5Y] >Bnn'3)+ ?M? ; (*;  ]X F  =0 (,$! ,$H 88GM@N?Yrs(;_c %NRN_N_+!65 :8 O`O KM-/K!LLLJM$}4M!24P P !--m< **488M+BC // /Q /  ]X F  =0        r>cb|jryd|_t|ddMtj|jj 5|jj ddddt|ddZtj|jj 5|jj dd|jgdddt|ddZtj|jj 5|jj dd|jgdddt|dddtj|jj 5|jj dd|jjgdddt|ddZtj|jj 5|jj dd|jgdddt|ddMtj|jj 5|jj ddddt|dd[tj|jj 5|jj dd|jgdddyy#1swYsxYw#1swYxYw#1swYxYw#1swY[xYw#1swYxYw#1swYxYw#1swYyxYw) NTrrrrrrr)rrr*rrr'rrrjrrrrrrrrs r<rzTFXGLMDecoderLayer.builds ::  4d + 7t~~223 +$$T* + 4/ 6 Bt88==> N))//tT^^0LM N 4 % 1txx}}- =dDNN;< = 4 % 1txx}}- BdDKK,?,?@A B 4+T 2 >t4499: J%%++T4,HI J 4 . :t00556 .!!''- . 42D 9 Et;;@@A Q,,22D$3OP Q Q F# + + N N = = B B J J . . Q QsT K%)K% )K233K?$)L  L%)L%K"%K/2K<?L  LL"%L.)rrrvrrNone)NNNNNNF)rrrrrrrrrrrrrzTuple[tf.Tensor] | Nonerrrz4Tuple[tf.Tensor, tf.Tensor, Tuple[Tuple[tf.Tensor]]]r)rrrrirrrrs@r<rr<sH,02637,07;26#(N  N )N  0 N !1 N * N %5N 0N !N  >N `Qr>rceZdZeZ d d fd Zd dZd dZ d dZdd dZ e d ddZ ddZ xZ S)TFXGLMMainLayerct||i|||_|j|_|j |_|jrtj|jnd|_ |||_ n2t|j|j|jd|_ d|_t!|j |jz|j|j|_t$j&j)|j*|_t-|j.Dcgc]}t1|d|c}|_|j2|_t$j&j5dd |_ycc}w) NrZ embed_tokensr&r)r6r7r8zlayers.r layer_normr)rhrir pad_token_idr8max_position_embeddingsmax_target_positionsscale_embeddingr(sqrtr embed_scalerr vocab_sizeoffsetr=_embed_positions_weightsrrlrmrnr, num_layersr layerdroprr)rtrrinputsrvirws r<rizTFXGLMMainLayer.__init__s< &+F+ !..$*$B$B!8>8N8N499V^^4TW  # ,D  2!!6>>43C3C.!D  (C 884;;F ..++) % ||++FNN; OTU[UfUfOgh!)&}Eh )),,99$\9Zis2F c|jSrrrts r<get_input_embeddingsz$TFXGLMMainLayer.get_input_embeddingss   r>c||_yrr)rtvalues r<set_input_embeddingsz$TFXGLMMainLayer.set_input_embeddingss !r>ct||tj|ddkDfdfd|St||d}|zS)Nr$rcSrrgcombined_attention_masksr<zATFXGLMMainLayer._prepare_decoder_attention_mask..s)@r>c.tjSr)r* ones_likersr<rzATFXGLMMainLayer._prepare_decoder_attention_mask..s",,WnJor>rV)rXr*condr^)rtrrLrEexpand_attention_maskrs @r<_prepare_decoder_attention_maskz/TFXGLMMainLayer._prepare_decoder_attention_masks`#4KAW"X"$'' Oa !@Bo#   !* * ,^[QS_ U$'>>>r>ch||jz }tj|j|d}|S)Nrr")rr*gatherr)rtrN positionss r<r%zTFXGLMMainLayer.embed_positionss/ # IId;;\PQR r>c | | n|jj} | | n|jj} | | n|jj} | | n|jj} | | t d|1t j|}t j|d|df}n&| t j| dd}n t d||ddjdnd}|1t jt j||d|zd}t j|dt|dg}| >t||jj|j||jz} |j!|||}||t#||d}|j%|}t j&| t j(|z}|j+|| }| rd nd}| rd nd}| r|d nd}| rd nd}d |fd |ffD]r\}}| t j,j/t|dt1|j2d |dt1|j2dt|ddtt5|j2D]z\}}| r||fz }t7j8dd}|r||j:kr6|||nd}||||||||nd|||nd|\}}}} | r|| fz }| sl||fz }|u||fz }||j=|}| r||fz }| r|nd}!| st?d||!|||fDStA||!|||S)NzDYou cannot specify both input_ids and inputs_embeds at the same timer$z5You have to specify either input_ids or inputs_embedsrrr"rr rrg head_maskcross_attn_head_maskzThe z should be specified for z layers, but it is for .rr)rrrrrrc3$K|]}|| ywrrg).0vs r< z'TFXGLMMainLayer.call..}s=s)last_hidden_statepast_key_valuesr attentionscross_attentions)!routput_attentionsoutput_hidden_states use_cacheuse_return_dictrpr*shaper/r.r,rrrrrrr^r%rAr-rnrrlenrl enumeraterandomuniformrrtupler)"rtrDrrNrrrrrrKrrr return_dictrrvrLrErrall_hidden_statesall_self_attnsall_cross_attentionsnext_decoder_cacheattn_mask_name attn_maskidx decoder_layerdropout_probabilityrlayer_self_attnlayer_cross_attnr next_caches" r<rzTFXGLMMainLayer.calls&2C1N-TXT_T_TqTq$8$D $++JjJj "+!6IDKKcd d  "((9-K 9r;r?.CDI  &((=1#26KTU UCRC^!3A!6!>/RCY1YZabLzz,Z 5Mb5Q0RS  *9d6G6G6R6R S --i84;K;KKM==nk[qr ! ,1G1S%12HR]^`Ra%b "((6  RZZ@9L  ]X F #7BD0d&7>A5D5P_S1VZNR_-&;'=3<3H3dI]Ii,@,Eos-S OM?,<>O"'8&::" ?"44(4(-=,??(9 @< 6  -!1 1 +4'$ '5FXlm  ;+&+%1   r>c|jryd|_t|dddtj|jj 5|jj dd|jjgdddt|ddMtj|jj 5|jj ddddt|ddK|jD];}tj|j 5|j dddd=yy#1swYxYw#1swYnxYw#1swYaxYw)NTrrrl) rrr*rrr'rrrrrl)rtrLlayers r<rzTFXGLMMainLayer.builds! ::  4t , 8t334 I%%tT4;;3F3F&GH I 4 . :t00556 .!!''- . 44 ( 4 &]]5::.&KK%&& & 5  I I . .&&s$ 3D9<EE9EEE r)rrrOptional[TFSharedEmbeddings]rvrrr)rr)rrrr)rrrLtf.TensorShaperErrr)rNnp.ndarray | tf.Tensor | NonerrNNNNNNNNNNNNNF rDTFModelInputType | NonerrrNrrrrrrrrrr4Optional[Tuple[Tuple[Union[np.ndarray, tf.Tensor]]]]rKrrrrrrrr rrrrvrrzDUnion[TFBaseModelOutputWithPastAndCrossAttentions, Tuple[tf.Tensor]])rrrr config_classrirrrr%rrrrrs@r<rrskLPT[ [0L[hk[ [:!"?(?$?!$ ?  ?" .28<6:?C@D37>BPT7;$(,0/3&*#(A *A 6A 4 A = A !> A 1A <A NA 5A "A *A -A $A !A !A " N#A A F &r>rceZdZeZdZy)TFXGLMPreTrainedModelmodelN)rrrrr!base_model_prefixrgr>r<r#r#s Lr>r#au This model inherits from [`TFPreTrainedModel`]. Check the superclass documentation for the generic methods the library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads etc.) This model is also a [keras.Model](https://www.tensorflow.org/api_docs/python/tf/keras/Model) subclass. Use it as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage and behavior. TensorFlow models and layers in `transformers` accept two formats as input: - having all inputs as keyword arguments (like PyTorch models), or - having all inputs as a list, tuple or dict in the first positional argument. The reason the second format is supported is that Keras methods prefer this format when passing inputs to models and layers. Because of this support, when using methods like `model.fit()` things should "just work" for you - just pass your inputs and labels in any format that `model.fit()` supports! If, however, you want to use the second format outside of Keras methods like `fit()` and `predict()`, such as when creating your own layers or models with the Keras `Functional` API, there are three possibilities you can use to gather all the input Tensors in the first positional argument: - a single Tensor with `input_ids` only and nothing else: `model(input_ids)` - a list of varying length with one or several input Tensors IN THE ORDER given in the docstring: `model([input_ids, attention_mask])` or `model([input_ids, attention_mask, token_type_ids])` - a dictionary with one or several input Tensors associated to the input names given in the docstring: `model({"input_ids": input_ids, "token_type_ids": token_type_ids})` Note that when creating models and layers with [subclassing](https://keras.io/guides/making_new_layers_and_models_via_subclassing/) then you don't need to worry about any of this, as you can just pass inputs like you would to any other Python function! Args: config ([`XGLMConfig`]): Model configuration class with all the parameters of the model. Initializing with a config file does not load the weights associated with the model, only the configuration. Check out the [`~TFPreTrainedModel.from_pretrained`] method to load the model weights. a Args: input_ids (`tf.Tensor` of shape `({0})`): Indices of input sequence tokens in the vocabulary. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) attention_mask (`tf.Tensor` of shape `({0})`, *optional*): Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. [What are attention masks?](../glossary#attention-mask) position_ids (`tf.Tensor` or `Numpy array` of shape `(batch_size, sequence_length)`, *optional*): Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`. [What are position IDs?](../glossary#position-ids) encoder_hidden_states (`tf.Tensor` of shape `(batch_size, encoder_sequence_length, hidden_size)`, *optional*): Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention of the decoder. encoder_attention_mask (`tf.Tensor` of shape `(batch_size, encoder_sequence_length)`, *optional*): Mask to avoid performing cross-attention on padding tokens indices of encoder input_ids. Mask values selected in `[0, 1]`: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. [What are attention masks?](../glossary#attention-mask) head_mask (`tf.Tensor` of shape `(num_layers, attention_heads)`, *optional*): Mask to nullify selected heads of the attention modules in the encoder. Mask values selected in `[0, 1]`: - 1 indicates the head is **not masked**, - 0 indicates the head is **masked**. cross_attn_head_mask (`tf.Tensor` of shape `(num_layers, attention_heads)`, *optional*): Mask to nullify selected heads of the cross-attention modules. Mask values selected in `[0, 1]`: - 1 indicates the head is **not masked**, - 0 indicates the head is **masked**. past_key_values (`Tuple[Tuple[tf.Tensor]]` of length `config.num_layers`) contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding. If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `decoder_input_ids` of shape `(batch_size, sequence_length)`. inputs_embeds (`tf.Tensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert `input_ids` indices into associated vectors than the model's internal embedding lookup matrix. use_cache (`bool`, *optional*, defaults to `True`): If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see `past_key_values`). Set to `False` during training, `True` during generation output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. This argument can be used only in eager mode, in graph mode the value in the config will be used instead. output_hidden_states (`bool`, *optional*): Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for more detail. This argument can be used only in eager mode, in graph mode the value in the config will be used instead. return_dict (`bool`, *optional*): Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. This argument can be used in eager mode, in graph mode the value will always be set to True. training (`bool`, *optional*, defaults to `False`): Whether or not to use the model in training mode (some modules like dropout modules have different behaviors between training and evaluation). z^The bare XGLM Model transformer outputting raw hidden-states without any specific head on top.ceZdZdZ d dfd Zeeeee e e  d d dZ ddZ xZS) TFXGLMModelz Transformer decoder consisting of *config.num_layers* layers. Each layer is a [`TFXGLMDecoderLayer`] Args: config: XGLMConfig embed_tokens: [TFSharedEmbeddings]: output embedding cRt||g|i|t||d|_y)Nr$rr')rhrirr$rtrrrrvrws r<rizTFXGLMModel.__init__s, 3&3F3$V,WU r> checkpoint output_typer!c B|j|||||||| | | | | | }|S)N) rDrrrrrrrKrrrr r)r$)rtrDrrNrrrrrrKrrrr rrvoutputss r<rzTFXGLMModel.call%sE2**)"7#9!5+'/!5#  r>c|jryd|_t|ddNtj|jj 5|jj ddddyy#1swYyxYw)NTr$)rrr*rr$r'rrs r<rzTFXGLMModel.buildPsg ::  4$ ' 3tzz/ '   & ' ' 4 ' 's A11A:r rrrrrrrvrrrrr)rrrrrirr XGLM_INPUTS_DOCSTRINGr _CHECKPOINT_FOR_DOCr_CONFIG_FOR_DOCrrrrs@r<r'r'sM PTV V0LV^aVmpV V*+@A&?$.28<6:?C@D37>BPT7;$(,0/3&*#("*"6"4 " = " !> "1"<"N"5"""*"-"$"!" !"" N#" B"H'r>r'z The XGLM Model transformer with a language modeling head on top (linear layer with weights tied to the input embeddings). c*eZdZdZddgZdgZ d dfd ZdZdZddZ e e e e eeeeee  d dd Zd d Zd ZxZS)TFXGLMForCausalLMr$zmodel.embed_positions.weightslm_head.weightct||g|i|t||d|_tj j |jdt|jd|_ ||_ y)Nr$r)Flm_head)rckernel_initializerr') rhrirr$rrlrsrrinit_stdr9rr*s r<rizTFXGLMForCausalLM.__init__jsk 3&3F3$V,WU ||))   .v? *   r>c|jSrr9rs r<get_output_embeddingsz'TFXGLMForCausalLM.get_output_embeddingsxs ||r>c||_yrr=)rtnew_embeddingss r<set_output_embeddingsz'TFXGLMForCausalLM.set_output_embeddings{s % r>c "|rtj|dddfd}|jdd}|jdd}|C|Atjj |dd}|rtj|dddfd}|||||dS)Nr$rNrT)r# exclusive)rDrrNrr)r*r.getr(rB)rtrrrrvrNrs r<prepare_inputs_for_generationz/TFXGLMForCausalLM.prepare_inputs_for_generation~s ^^F1b5M26Fzz.$7 $4d;  %,*>77>>.rT>RL!~~l1b5.A2F  ,(."   r>)r-r!r+c "|j||||||||| | | | ||}|d}|j|}d}| tj| ddddftj| j ddftj |jj| jgd} |j| |}|s|f|ddz}||f|zS|St|||j|j|j|jS)a labels (`np.ndarray` or `tf.Tensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100` are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]` )rDrrNrrrrrrKrrrr rrNrr$r")losslogitsrrrr)r$r9r*r0fillrrArrr!hf_compute_lossrrrrr)rtrDrrNrrrrrrKlabelsrrrr rrvr/r lm_logitsrGoutputs r<rzTFXGLMForCausalLM.calls4D**)%"7#9!5+'/!5#    LL/   YY12a!(m noF'' :D\GABK/F)-)9TGf$ Ev E2#33!//))$55   r>c|jryd|_t|ddMtj|jj 5|jj ddddt|ddetj|jj 5|jj dd|jjgdddyy#1swY|xYw#1swYyxYw)NTr$r9) rrr*rr$r'rr9r hidden_sizers r<rzTFXGLMForCausalLM.builds ::  4$ ' 3tzz/ '   & ' 4D ) 5t||001 J ""D$ 0G0G#HI J J 6 ' ' J Js C"%3C."C+.C7c|dk(r|dfS|fS)Nr7zmodel.embed_tokens.weightrg)rt tf_weights r<tf_to_pt_weight_renamez(TFXGLMForCausalLM.tf_to_pt_weight_renames ( (99 9< r>rr1)NN)NNNNNNNNNNNNNNF)"rDrrrrNrrrrrrrrrrr rKrrKrrrrrrrr rrrrvrrzrArErr r2r rr4r r3rrrRrrs@r<r6r6Ys ('# ) PT   0L ^a mp   & **+@A+N]lm&7$.28<6:?C@D37>BPT7;04$(,0/3&*#(!A *A 6A 4 A = A !> A 1A <A NA 5A .A "A *A -A $A !!A "#A $ F%A nBA F J r>r6)r6rr7rr8 Optional[int]rr)rDrrErr8rUrr)rKrrErr8rUrr)r)rTrrErr)rFrrVrU)>r __future__rr(rtypingrrrrnumpynp tensorflowr*activations_tfr file_utilsr r r r modeling_tf_outputsrrmodeling_tf_utilsrrrrrrrrtf_utilsrrrutilsrconfiguration_xglmr get_loggerrloggerr3r4rRr=rHrOrXr^rlLayerr`rrr#XGLM_START_DOCSTRINGr2r'r6rgr>r<rfs" ../ t   SR*   H %*4. F F25 FDQ F Fgg69gHUgg$;$ 6gBell((gBTJQ++JQZM&ell((M&M&` - 'RFRdA''A' A'H M -/KM M r>