sgӚdZddlmZddlmZmZmZddlZddl Z ddl m Z m Z mZddlmZmZmZmZmZmZmZmZddlmZmZmZdd lmZmZmZm Z d d l!m"Z"e jFe$Z%d Z&d Z'dZ(dZ)d(dZ*GddejVjXZ-GddejVjXZ.GddejVjXZ/eGddejVjXZ0GddeZ1dZ2dZ3ede2Gdde1Z4Gd d!ejVjXZ5ed"e2Gd#d$e1eZ6ed%e2Gd&d'e1eZ7y))zTF 2.0 CTRL model.) annotations)OptionalTupleUnionN)TFBaseModelOutputWithPastTFCausalLMOutputWithPastTFSequenceClassifierOutput)TFCausalLanguageModelingLossTFModelInputTypeTFPreTrainedModelTFSequenceClassificationLossget_initializerkeraskeras_serializable unpack_inputs)check_embeddings_within_bounds shape_liststable_softmax)add_code_sample_docstringsadd_start_docstrings%add_start_docstrings_to_model_forwardlogging) CTRLConfigzSalesforce/ctrlrcPdtjdd|dzz|z z }||zS)Nri')nppower)posi d_model_size angle_ratess \/var/www/html/venv/lib/python3.12/site-packages/transformers/models/ctrl/modeling_tf_ctrl.py angle_defnr%/s/bhhuqAF||&CDDK  cttj|ddtjftj|tjddf|}tj|dddddf}tj |dddddf}t jtj||gd}|S)Nrrraxis) r%rarangenewaxissincostfconvert_to_tensor concatenate)positionr" angle_radssinescosines pos_encodings r$positional_encodingr74sBIIh/2:: > ,@WXZXbXbdeXe@fhtuJ FF:aAg& 'EffZ14a4()G''w7Gb(QRL r&ctj||d}tjt|d|j}|tj j |z }|'|tj|dz|jz }|&tj||j}||z}t|d} || |z} tj| |} | | fS)NT transpose_br(dtyper))r/matmulcastrr<mathsqrtr) qkvmaskattention_mask head_mask matmul_qkdkscaled_attention_logitsattention_weightsoutputs r$scaled_dot_product_attentionrM?s !QD1I Ar")// :B'"'',,r*:: 2774$;>U>[>[#\\!7N7T7TU"9N"J&'>RH- 9 YY(! ,F $ $$r&c6eZdZdfd ZdZddZddZxZS)TFMultiHeadAttentionc t|di|||_||_||_t ||jz |_tjj|d|_ tjj|d|_ tjj|d|_ tjj|d|_ y)NWqnameWkWvdense)super__init__ num_headsr"output_attentionsintdepthrlayersDenserQrTrUrV)selfr"rZr[kwargs __class__s r$rYzTFMultiHeadAttention.__init__Zs "6""(!267 ,,$$\$=,,$$\$=,,$$\$=\\'' 7'C r&ctj||d|j|jf}tj|gdS)Nr(rrrrperm)r/reshaperZr] transpose)r`x batch_sizes r$split_into_headsz%TFMultiHeadAttention.split_into_headshs4 JJq:r4>>4::F G||AL11r&c t|d} |j|}|j|}|j|}|j || }|j || }|j || }|Lt j |d\} } t j| |fd}t j| |fd}|rt j||fd}nd}t||||||}t j|dgd}|d}t j|| d|jf}|j|}||f}| r||fz}|S) Nrr)Nrdrerr()rrQrTrUrkr/unstackconcatstackrMrhrgr"rV)r`rDrCrBrE layer_pastrFrG use_cacher[trainingrjpast_key past_valuepresentrLscaled_attentionattnoriginal_size_attentionoutputss r$callzTFMultiHeadAttention.calllsK]1% GGAJ GGAJ GGAJ  ! !!Z 0  ! !!Z 0  ! !!Z 0  !#%::jq#A Hj 8Q-b1A :q/3A hh1vA.GG-aAt^YW<<q Eay"$**-= BPTPaPa?b"c347# 'Gr&c|jryd|_t|ddZtj|jj 5|jj dd|jgdddt|ddZtj|jj 5|jj dd|jgdddt|ddZtj|jj 5|jj dd|jgdddt|dd[tj|jj 5|jj dd|jgdddyy#1swYAxYw#1swYxYw#1swYxYw#1swYyxYw)NTrQrTrUrV) builtgetattrr/ name_scoperQrSbuildr"rTrUrVr` input_shapes r$rzTFMultiHeadAttention.buildsx ::  4t $ 0tww||, ? tT4+<+<=> ? 4t $ 0tww||, ? tT4+<+<=> ? 4t $ 0tww||, ? tT4+<+<=> ? 4$ ' 3tzz/ B   $d.?.?!@A B B 4 ? ? ? ? ? ? B Bs0 )F32)G)G )G3F=G  GG!Frn)__name__ __module__ __qualname__rYrkr|r __classcell__rbs@r$rOrOYs D2BBr&rOc.eZdZfdZddZddZxZS)TFPointWiseFeedForwardLayerc t|di|tjj |dd|_tjj |d|_||_||_y)Nrelu0) activationrS2rRrW) rXrYrr^r_dense_0dense_2r"dff)r`r"rrarbs r$rYz$TFPointWiseFeedForwardLayer.__init__sZ "6"||))#&s)K ||)),S)A (r&cJ|j|}|j|}|Srn)rr)r`inputs trainabledense_0_outputdense_2_outputs r$r|z TFPointWiseFeedForwardLayer.calls$f-n5r&c|jryd|_t|ddZtj|jj 5|jj dd|jgdddt|dd[tj|jj 5|jj dd|jgdddyy#1swYrxYw#1swYyxYw)NTrr) r~rr/rrrSrr"rrrs r$rz!TFPointWiseFeedForwardLayer.builds ::  4D ) 5t||001 D ""D$0A0A#BC D 4D ) 5t||001 ; ""D$#9: ; ; 6 D D ; ;s )C%2)C1%C.1C:rrnrrrrYr|rrrs@r$rrs ;r&rc2eZdZ dfd ZddZddZxZS)TFEncoderLayerc t|di|||_t|||jd|_t ||d|_tjj|d|_ tjj|d|_ tjj||_ tjj||_||_y) Nmulti_head_attention)r[rSffnrR layernorm1epsilonrS layernorm2rW)rXrYr[rOrrrrr^LayerNormalizationrrDropoutdropout1dropout2r") r`r"rZrratelayer_norm_epsilonr[rarbs r$rYzTFEncoderLayer.__init__s "6"!2$8 )t7M7MTj% !/|SuM,,99BT[g9h,,99BT[g9h ,,T2  ,,T2 (r&c |j|} |j| | | ||||||| } | d} |j| |} || z} |j| } |j | }|j ||}| |z} | f| ddz}|S)Nrtrr)rrrrrr)r`rirErrrFrGrsr[rtnormed attn_outputs attn_outputout1out2 ffn_outputr{s r$r|zTFEncoderLayer.calls#00         1  #1o mmK(mC ;t$XXd^ ]]:]A j 'L,,r&c|jryd|_t|ddMtj|jj 5|jj ddddt|ddMtj|jj 5|jj ddddt|ddZtj|jj 5|jj dd|jgdddt|dd[tj|jj 5|jj dd|jgdddyy#1swY4xYw#1swYxYw#1swYxYw#1swYyxYw)NTrrrr) r~rr/rrrSrrrr"rrs r$rzTFEncoderLayer.buildsq ::  4/ 6 Bt88==> 6))//5 6 4 % 1txx}}- %t$ % 4t , 8t334 G%%tT43D3D&EF G 4t , 8t334 G%%tT43D3D&EF G G 9 6 6 % % G G G Gs0 F%F&?)F2&)F>F#&F/2F;>G)g?gư>Frrnrrs@r$rrsaf)&4Gr&rceZdZeZfdZdZdZdZe d ddZ d dZ xZ S) TFCTRLMainLayerc t|di|||_|j|_|j|_|j |_|j |_|j|_ |j|_ t|j|j|_tj j#|j$|jt'|j(d|_tj j-|j.|_t3|jDcgc]S}t5|j|j6|j8|j:|j<|jd|Uc}|_tj jA|j<d|_!ycc}w)Nw) input_dim output_dimembeddings_initializerrSzh_._rR layernormrrW)"rXrYconfigoutput_hidden_statesr[rsuse_return_dict return_dictn_embdr"n_layer num_layersr7 n_positionsr6rr^ Embedding vocab_sizerinitializer_rangerr embd_pdropdropoutrangern_headr resid_pdroprhrr)r`rrar!rbs r$rYzTFCTRLMainLayer.__init__sk "6" $*$?$?!!'!9!9))!11"MM ../0B0BDDUDUV''''}}#263K3K#L ( ||++F,=,=> 6>>*      ""))&&A3Z   88AZAZal8m  s7AGc|jSrnrr`s r$get_input_embeddingsz$TFCTRLMainLayer.get_input_embeddings"s vv r&c||_yrnr)r`new_embeddingss r$set_input_embeddingsz$TFCTRLMainLayer.set_input_embeddings%s r&ct)zv Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} )NotImplementedError)r`heads_to_prunes r$ _prune_headszTFCTRLMainLayer._prune_heads(s "!r&c h |'| |ddddf}| |ddddf}| |ddddf}| | td|'t|} tj|d| dg}n|t|dd} n td|d}dgt |j z}nt|ddd}|\tj tj|| d|ztjd}tj|| ddg}|tj|| ddd| d|zf}tjd }tjd }tj||j}tjtj|||}|tdg|j z}|tj|dt|dg}|j#|}|tj$j'tj|j(|jz}ntjd }tj|dt|dg}|1t+||j"j,|j#|}| d}dtj.j1tj2||fddz }|tj$j'tj|j(|jz}tj4|j6|}tj||j}||z|z}|j9|| }| t|dgz}|rd nd}| rd nd}| rd nd}t;t=|j |D]S\}\}}| r|tj||fz}||||||||| | }|dd\}}|r||fz}| sK||dfz}U|j?|}tj||}| r||fz}| r/| dddgzt|dddztAfd|D}| stAd||||fDStC||||S)Nr(zDYou cannot specify both input_ids and inputs_embeds at the same timez5You have to specify either input_ids or inputs_embedsrrmr;r)rg?r=grrWrc3JK|]}tj|ywrn)r/rg).0tattention_output_shapes r$ z'TFCTRLMainLayer.call..s"aQ2::a1G#H"as #c3&K|] }|| ywrnrW)rrDs r$rz'TFCTRLMainLayer.call..srqdedqrs)last_hidden_statepast_key_values hidden_states attentions)" ValueErrorrr/rglenr expand_dimsrint32tileconstantr?r<multiplysubtractrrrr@rAr"rrlinalg band_partonesgatherr6r enumerateziprtupler) r` input_idsrrFtoken_type_ids position_idsrG inputs_embedsrsr[rrrtr past_lengthone_cstten_thousand_csttoken_type_embedsseq_lenrE pos_embedsr output_shapepresentsall_hidden_statesall_attentionsr!rrrr{rwrs @r$r|zTFCTRLMainLayer.call.s$  &$%af- ( -af 5 )!/23!7  ]%>cd d  "$Y/K 9r;r?.CDI  &$]3CR8KTU U  "K#fs466{2O$_Q%7%:;B?K  >>"((; BR]@]egemem*nuvwL77<+a.!1DEL  %  ZZQA{[\~`kOk8lmNkk#&G!{{84 WW^7==IN[[Wn)MO_`N  % %0I  %ZZZ=WXZ=[8\]N $~ 6  bggd6G6GO`OfOf.g!h h  " C 0 zz,Z 5Mb5Q0RS  *9dff6F6F G FF9-Mb/299&&rww/A'BBJJbggd.?.?ATAT&UVV YYt00,? WWZ/@/F/FG % 25FF  ]X F "j&?&C%DD "2"6BD0d"+C,H"I @ A:#$5MS_9`8b$b!! !! G&-Ra[ "M7#wj0 !/71:-!?' @*}5  =,?  1]4D D  %0"%5%OQ_$`rr r(+$+%   r&c|jryd|_t|ddMtj|jj 5|jj ddddt|dddtj|jj 5|jj dd|jjgdddt|ddK|jD];}tj|j 5|j dddd=yy#1swYxYw#1swYnxYw#1swYaxYw)NTrrr) r~rr/rrrSrrrrr)r`rlayers r$rzTFCTRLMainLayer.builds ::  4d # /tvv{{+ # T" # 4d + 7t~~223 G$$dD$++2D2D%EF G 4d # / &]]5::.&KK%&& & 0  # # G G&&s$ D9%3EE9EEE NNNNNNNNNNNFrTFModelInputType | Noner4Optional[Tuple[Tuple[Union[np.ndarray, tf.Tensor]]]]rFnp.ndarray | tf.Tensor | NonerrrrrGrrrrsOptional[bool]r[rrrrrrtrreturnz'Union[Tuple, TFBaseModelOutputWithPast]rn) rrrr config_classrYrrrrr|rrrs@r$rrsL"nH " .2PT8<8<6:377;$(,0/3&*#(J *J NJ 6 J 6 J 4 J 1J 5J "J *J -J $J !J  1J J X &r&rceZdZdZeZdZy)TFCTRLPreTrainedModelz An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained models. transformerN)rrr__doc__rrbase_model_prefixrWr&r$r r s L%r&r az This model inherits from [`TFPreTrainedModel`]. Check the superclass documentation for the generic methods the library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads etc.) This model is also a [keras.Model](https://www.tensorflow.org/api_docs/python/tf/keras/Model) subclass. Use it as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage and behavior. TensorFlow models and layers in `transformers` accept two formats as input: - having all inputs as keyword arguments (like PyTorch models), or - having all inputs as a list, tuple or dict in the first positional argument. The reason the second format is supported is that Keras methods prefer this format when passing inputs to models and layers. Because of this support, when using methods like `model.fit()` things should "just work" for you - just pass your inputs and labels in any format that `model.fit()` supports! If, however, you want to use the second format outside of Keras methods like `fit()` and `predict()`, such as when creating your own layers or models with the Keras `Functional` API, there are three possibilities you can use to gather all the input Tensors in the first positional argument: - a single Tensor with `input_ids` only and nothing else: `model(input_ids)` - a list of varying length with one or several input Tensors IN THE ORDER given in the docstring: `model([input_ids, attention_mask])` or `model([input_ids, attention_mask, token_type_ids])` - a dictionary with one or several input Tensors associated to the input names given in the docstring: `model({"input_ids": input_ids, "token_type_ids": token_type_ids})` Note that when creating models and layers with [subclassing](https://keras.io/guides/making_new_layers_and_models_via_subclassing/) then you don't need to worry about any of this, as you can just pass inputs like you would to any other Python function! Parameters: config ([`CTRLConfig`]): Model configuration class with all the parameters of the model. Initializing with a config file does not load the weights associated with the model, only the configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. aZ Args: input_ids (`Numpy array` or `tf.Tensor` of shape `(batch_size, input_ids_length)`): `input_ids_length` = `sequence_length` if `past` is `None` else `past[0].shape[-2]` (`sequence_length` of input past key value states). Indices of input sequence tokens in the vocabulary. If `past` is used, only input IDs that do not have their past calculated should be passed as `input_ids`. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.__call__`] and [`PreTrainedTokenizer.encode`] for details. [What are input IDs?](../glossary#input-ids) past (`List[tf.Tensor]` of length `config.n_layers`): Contains pre-computed hidden-states (key and values in the attention blocks) as computed by the model (see `past` output below). Can be used to speed up sequential decoding. The token ids which have their past given to this model should not be passed as input ids as they have already been computed. attention_mask (`tf.Tensor` or `Numpy array` of shape `(batch_size, sequence_length)`, *optional*): Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. [What are attention masks?](../glossary#attention-mask) token_type_ids (`tf.Tensor` or `Numpy array` of shape `(batch_size, sequence_length)`, *optional*): Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`: - 0 corresponds to a *sentence A* token, - 1 corresponds to a *sentence B* token. [What are token type IDs?](../glossary#token-type-ids) position_ids (`tf.Tensor` or `Numpy array` of shape `(batch_size, sequence_length)`, *optional*): Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`. [What are position IDs?](../glossary#position-ids) head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - 1 indicates the head is **not masked**, - 0 indicates the head is **masked**. inputs_embeds (`tf.Tensor` or `Numpy array` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert `input_ids` indices into associated vectors than the model's internal embedding lookup matrix. use_cache (`bool`, *optional*): If set to `True`, `past` key value states are returned and can be used to speed up decoding (see `past`). output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. This argument can be used only in eager mode, in graph mode the value in the config will be used instead. output_hidden_states (`bool`, *optional*): Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for more detail. This argument can be used only in eager mode, in graph mode the value in the config will be used instead. return_dict (`bool`, *optional*): Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. This argument can be used in eager mode, in graph mode the value will always be set to True. training (`bool`, *optional*, defaults to `False`): Whether or not to use the model in training mode (some modules like dropout modules have different behaviors between training and evaluation). z^The bare CTRL Model transformer outputting raw hidden-states without any specific head on top.ceZdZfdZeeeeee e  d ddZ ddZ xZ S) TFCTRLModelcPt||g|i|t|d|_y)Nr rR)rXrYrr r`rrrarbs r$rYzTFCTRLModel.__init__Gs) 3&3F3*6 Fr& checkpoint output_typerc @|j||||||||| | | |  } | S)N rrrFrrrGrrsr[rrrt)r )r`rrrFrrrGrrsr[rrrtr{s r$r|zTFCTRLModel.callKsD,""+))%'/!5##  r&c|jryd|_t|ddNtj|jj 5|jj ddddyy#1swYyxYw)NTr )r~rr/rr rSrrs r$rzTFCTRLModel.buildqsm ::  4 - 9t//445 -  &&t, - - : - -s A11A:rrrn)rrrrYrrCTRL_INPUTS_DOCSTRINGr_CHECKPOINT_FOR_DOCr_CONFIG_FOR_DOCr|rrrs@r$rrBs G*+@A&-$.2PT8<8<6:377;$(,0/3&*#(*N6  6  4 15"*-$! 1 B>-r&rc2eZdZdZfdZfdZdZxZS)TFCTRLBiasLayerz Bias as a layer. It is used for serialization purposes: `keras.Model.save_weights` stores on a per-layer basis, so all weights have to be registered in a layer. c Rt|dd|i|||_||_||_y)NrSrW)rXrYshape initializerr)r`rrrrSrarbs r$rYzTFCTRLBiasLayer.__init__s. -d-f- &"r&c|jd|j|j|j|_t ||y)NbiasrSrrr) add_weightrrrr!rXr)r`rrbs r$rzTFCTRLBiasLayer.builds@OOtzzt7G7GSWSaSa$    k"r&c ||jzSrn)r!)r`ris r$r|zTFCTRLBiasLayer.calls499}r&)rrrr rYrr|rrs@r$rrzs # # r&rz The CTRL Model transformer with a language modeling head on top (linear layer with weights tied to the input embeddings). ceZdZfdZdZdZdZdZd dZe e e e e ee d d dZd d ZxZS)TFCTRLLMHeadModelct||g|i|t|d|_t dd|j gdd|_y)Nr rRlm_headrzerosTr")rXrYrr rr bias_layerrs r$rYzTFCTRLLMHeadModel.__init__sJ 3&3F3*6 F)1f&7&7"8gY] r&c"|jSrn)rrs r$get_output_embeddingsz'TFCTRLLMHeadModel.get_output_embeddingss((**r&c&|j|yrn)r)r`values r$set_output_embeddingsz'TFCTRLLMHeadModel.set_output_embeddingss !!%(r&c2d|jjiS)N lm_head.bias)r*r!rs r$get_biaszTFCTRLLMHeadModel.get_biass 4 455r&c|djd}tdd|gdd|_|jjd|jjj |dy)Nr1r(final_logits_biasrr)Tr")rrr*rr!assign)r`r.rs r$set_biaszTFCTRLLMHeadModel.set_biassb>*004 )$Q O\`  d# ##E.$9:r&c |jdd}|r77>>.rT>RL!~~l1b5.A2F  ,(.",   r&rc|j||||||||| | | |  }|d}tj||jjjd}|j |}d}| (|ddddf}| ddddf} |j | |}| s|f|ddz}||f|zS|St|||j|j|jS) labels (`tf.Tensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the cross entropy classification loss. Indices should be in `[0, ..., config.vocab_size - 1]`. rrTr9Nr(r)losslogitsrrr) r r/r>rweightsr*hf_compute_lossr rrr)r`rrrFrrrGrrsr[rrlabelsrttransformer_outputsrr?r>shifted_logitsrLs r$r|zTFCTRLLMHeadModel.calls8#..+))%'/!5#/  ,A. =$*:*:*<*<*D*DRVW(  #AssF^NAqrE]F''?DY!4QR!88F)-)9TGf$ Ev E'/??-;;*55   r&c|jryd|_t|ddMtj|jj 5|jj ddddt|ddNtj|jj 5|jj ddddyy#1swYexYw#1swYyxYw)NTr r*)r~rr/rr rSrr*rs r$rzTFCTRLLMHeadModel.build s ::  4 - 9t//445 -  &&t, - 4t , 8t334 ,%%d+ , , 9 - - , ,s C %C CC NN NNNNNNNNNNNNF)rrrrrFrrrrrrGrrrrsrr[rrrrrrBrrtrrz&Union[Tuple, TFCausalLMOutputWithPast]rn)rrrrYr,r/r2r6r;rrrrrr rr|rrrs@r$r&r&s +)6; 2*+@A&,$.2PT8<8<6:377;$(,0/3&*04#(8 *8 N8 6 8 6 8 4 8 18 58 "8 *8 -8 $8 .8 !8  08 B8 t ,r&r&a The CTRL Model transformer with a sequence classification head on top (linear layer). [`TFCTRLForSequenceClassification`] uses the last token in order to do the classification, as other causal models (e.g. GPT-1, GPT-2) do. Since it does classification on the last token, it requires to know the position of the last token. If a `pad_token_id` is defined in the configuration, it finds the last token that is not a padding token in each row. If no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in each row of the batch). ceZdZfdZdZeeeee e e  d ddZ ddZ xZS) TFCTRLForSequenceClassificationc t||g|i||j|_tjj |jt |jdd|_t|d|_ ||_ y)N classifierF)kernel_initializerrSuse_biasr rR) rXrY num_labelsrr^r_rrrKrr rrs r$rYz(TFCTRLForSequenceClassification.__init__(sw 3&3F3 ++,,,,   .v/G/GH -  +6 F r&cXtjd|jjS)Nz}Sequence classification models do not have output embeddings. `.get_output_embeddings` will be removed in transformers v4.32.)loggerwarningr rrs r$r,z5TFCTRLForSequenceClassification.get_output_embeddings4s' % !!!r&rc@|j||||||||| | | |  }|d}|j|}d}|jjd}n|t j t j tjj||jj|jddz }t j|dk\||jddz }t j||dd}n.d}tj|jj dd}| |t#|dd \}}nt#|dd \}}|jj|dk7r t%d t j&|s |d||f}|j)t j*| ddgt j*|d|j,g}||n|}| s|f|ddz}||f|zS|St/|||j0|j2 S) r=rrNr(r)r) batch_dimsr*z will not detect padding tokens in `inputs_embeds`. Results may be unexpected if using padding tokens in conjunction with `inputs_embeds.`rz=Cannot handle batch sizes > 1 if no padding token is defined.)r>r?rr)r rKr pad_token_idr/argmaxr?r@equalr<whererrrP warning_oncerbrrr is_tensorrArgrNr rr)r`rrrFrrrGrrsr[rrrBrtrCrr? in_logitssequence_lengthsr>rjsequence_length pooled_logitsrLs r$r|z$TFCTRLForSequenceClassification.call<sQ:#..+))%'/!5#/  ,A. / ;; # # +! $IIbggbggmmIt{{?W?W&XZcZiZijqst!$&88,<,ACSU^UdUdegUhklUl#m IIf.>1STU #% ##~~../0^^  $.8.CBQ.G+ O.8.G.K+ O{{''/J!O !`aa<< 01"1Z<1A#AB '' 6B7(CRZZPY\^`d`o`o[pEqrD%.%:  #%(;AB(??F)-)9TGf$ Ev E) -;;*55   r&c|jryd|_t|dddtj|jj 5|jj dd|jjgdddt|ddNtj|jj 5|jj ddddyy#1swYexYw#1swYyxYw)NTrKr ) r~rr/rrKrSrrrr rs r$rz%TFCTRLForSequenceClassification.builds ::  4t , 8t334 H%%tT4;;3E3E&FG H 4 - 9t//445 -  &&t, - - : H H - -s 3C"<C."C+.C7rG)rrrrrFrrrrrrGrrrrsrr[rrrrrrBrrtrrz(Union[Tuple, TFSequenceClassifierOutput]rn)rrrrYr,rrrrrr rr|rrrs@r$rIrIs "*+@A&.$.2PT8<8<6:377;$(,0/3&*04#(R *R NR 6 R 6 R 4 R 1R 5R "R *R -R $R .R !R  2R BR h -r&rIrF)8r  __future__rtypingrrrnumpyr tensorflowr/modeling_tf_outputsrr r modeling_tf_utilsr r r rrrrrtf_utilsrrrutilsrrrrconfiguration_ctrlr get_loggerrrPrrr%r7rMr^LayerrOrrrr CTRL_START_DOCSTRINGrrrr&rIrWr&r$rks "))rr   SRuu*   H %' %4CB5<<--CBL;%,,"4"4;6=GU\\''=G@M&ell((M&M&`&-&(T@Fd1-'1- 1-hell((, ~,-/K~,~,B y-&;=Yy-y-r&