sg׻dZddlmZddlmZmZmZddlZddl Z ddl m Z ddl mZmZmZddlmZmZmZmZdd lmZmZmZmZmZmZmZmZmZm Z dd l!m"Z"m#Z#m$Z$dd l%m&Z&d d l'm(Z(e&jRe*Z+dZ,dZ-d+dZ.d,dZ/d-dZ0GddejbjdZ3GddejbjdZ4GddejbjdZ5eGddejbjdZ6GddeZ7dZ8dZ9ede8Gd d!e7Z:ed"e8Gd#d$e7eZ;ed%e8Gd&d'e7eZ<ed(e8Gd)d*e7eZ=y).zTF 2.0 GPT-J model.) annotations)OptionalTupleUnionN)get_tf_activation)add_code_sample_docstringsadd_start_docstrings%add_start_docstrings_to_model_forward)TFBaseModelOutputWithPastTFCausalLMOutputWithPastTFQuestionAnsweringModelOutput"TFSequenceClassifierOutputWithPast) TFCausalLanguageModelingLossTFModelInputTypeTFPreTrainedModelTFQuestionAnsweringLossTFSequenceClassificationLossTFSharedEmbeddingsget_initializerkeraskeras_serializable unpack_inputs)check_embeddings_within_bounds shape_liststable_softmax)logging) GPTJConfigzEleutherAI/gpt-j-6Brc tjddtjd|d|z zz tj}tjtjdtj|tj|tj}tj |tj |}}tj||fd}|S) N?i'rz i , j -> i jdtyperaxis)tfcastrangefloat32einsumsincosconcat)num_posdiminv_freq sinusoid_inpr,r-outs \/var/www/html/venv/lib/python3.12/site-packages/transformers/models/gptj/modeling_tf_gptj.pycreate_sinusoidal_positionsr5;swwseC(;c(ABCRZZPH77299^RXXgRZZ5XZbcegeoeopLvvl#RVVL%9C ))S#JQ 'C Jc tj|dddddddddf |dddddddddffd}t|ddtjj t|ddgz}tj ||}|S)Nrr"r%)r'stackrmath reduce_prodreshape)xrotate_half_tensor new_shapes r4rotate_every_tworACsAaAqt!tm$4#4a1a1 o"FRP-.s3rww7J7J:VhKijljmKn7o6ppI$6 B r6c|\}}tj|dddddddfdd}tj|dddddddfdd}||zt||zzS)Nr"r)r'repeatrA)tensorsincossin_poscos_poss r4apply_rotary_pos_embrHJseGWii1dA .15Gii1dA .15G W !1&!9G!C DDr6ceZdZd fd Zd dZed dZd dZd dZ d ddZ d ddZ ddZ xZ S)TFGPTJAttentionc <t|d i||j|_|j|_|j|jz|_|j |jz|jk7r&t d|jd|jd|j dz|_|j|_tjj|j|_ tjj|j|_tjj!|jdt#|j$d|_tjj!|jdt#|j$d|_tjj!|jdt#|j$d |_tjj!|jdt#|j$d |_|j.|_t3j4t3j6t2j8j:j=t3j>|j0|j0ft2j@d d |j0|j0f|_!|jxs |j}tE|j0||_#y) NzEembed_dim must be divisible by num_attention_heads (got `embed_dim`: z and `num_attention_heads`: z).g?Fq_projuse_biaskernel_initializernamek_projv_projout_projr)$super__init__ hidden_size embed_dimnum_attention_headshead_dim ValueError scale_attn rotary_dimrlayersDropout attn_pdrop attn_dropout resid_pdrop resid_dropoutDenserinitializer_rangerLrQrRrSmax_position_embeddings max_positionsr'r=r( experimentalnumpytrilonesint8lower_triangle_maskr5embed_positions)selfconfigkwargs pos_embd_dim __class__s r4rVzTFGPTJAttention.__init__Rsr "6"++#)#=#= $*B*BB ==433 3t~~ EWX\XfXfWgh++/+C+C*DBH --, ++!LL001B1BC"\\11&2D2DEll(( NN.v/G/GH )  ll(( NN.v/G/GH )  ll(( NN.v/G/GH )   ** NN.v/G/GH +  $;;#%:: GGBOO))..rww8J8JDL^L^7_/`acecjcj k 4%%t'9'9 :$  8$.. :4;M;M|\r6ctj|jdddd||z |d|ftjSN)r'r(rmbool)ro key_length query_lengths r4get_causal_maskzTFGPTJAttention.get_causal_masks@wwt//1j<6OR\6\^i_i^i0ijlnlslsttr6cTtjtjd|S)Nge)r'r(constantr#s r4get_masked_biaszTFGPTJAttention.get_masked_biasswwr{{4(%00r6cxt|dd|j|jgz}tj||}|r|St t|dk(rtj |dSt t|dk(rtj |dStdt t|)zO Splits hidden dim into attn_head_size and num_attention_heads Nr8rr"rrrrrr"r~3Input tensor rank should be one of [4, 5], but is: )rrYrZr'r=len transposer[)ro hidden_statesrotaryr@s r4 _split_headszTFGPTJAttention._split_headss}-cr2d6N6NPTP]P]5^^  =)<  z-( )Q .<< |< < z-( )Q .<< ? ?NsS]^kSlOmNnoppr6cxtt|dk(rtj|d}nNtt|dk(rtj|d}n t dtt|t|dd|j |j zgz}tj||S)zR Merges attn_head_size dim and num_attn_heads dim into hidden dim r~rrrrNr9)rrr'rr[rYrZr=)rorr@s r4 _merge_headszTFGPTJAttention._merge_headss z-( )Q .LL EM M* +q 0LLHMRSVWaboWpSqRrst t}-cr2d6N6NQUQ^Q^6^5__ zz-33r6cht|dt|d}}|j||}tj|tj}tj|tj}tj ||d} tj || |j| j} | |jz } || |z} t| d} tj| |j} |j| } || |z} tj | |} | | fS)Nr9T) transpose_br8r%) rryr'r(r*matmulwherer|r$r\rra) roquerykeyvalueattention_mask head_maskrxrw causal_mask attn_weights attn_outputs r4_attnzTFGPTJAttention._attns $.e#4R#8*S/":Mj **:|D rzz*ggc2::&yy> xx \4;O;OP\PbPb;cd #doo5  %'.8L%l< ww|U[[9 ((6   ')3Lii e4 L((r6c|j|}|j|} |j|} |j|d}|j| d} |j| d} t j t j |j|d|j} t j| dd} |j| ddddddd|jf} | dddddd|jdf} |ddddddd|jf}|dddddd|jdf}t| | } t|| }t j| | fd} t j||fd}nt| | } t|| }t j| d} t j|d}|<|d}|d}t j|| fd } t j|| fd } |dur| | f}nd}|j|| | ||\}}|j|}|j!|}|j#|}||f}|r||fz }|S) NTFrr%r"r8rrr9)rLrQrRrr'r(gatherrnr$splitr]rHr.rrrrSrc)ror layer_pastr position_idsr use_cacheoutput_attentionsrrrrEk_rotk_passq_rotq_passpast_key past_valuepresentrroutputss r4callzTFGPTJAttention.callsQ M*kk-( M*!!%.T*!!%/4#7#7ANP]PcPcd&!"- ?? &1a!24??!223EAq$//"334F!Q#4T__#445E1aDOO$556F(7E(7E))UFO"5CIIufoB7E&sF3C(7Ell3 - UL1  !!!}H#AJ))XsO"5CIIz51;E  ElGG%)JJuc5.R[$\! \'' 4 mmK0 ((5 (    &Gr6c|jryd|_t|ddZtj|jj 5|jj dd|jgdddt|ddZtj|jj 5|jj dd|jgdddt|ddZtj|jj 5|jj dd|jgdddt|dd[tj|jj 5|jj dd|jgdddyy#1swYAxYw#1swYxYw#1swYxYw#1swYyxYw)NTrLrQrRrS) builtgetattrr' name_scoperLrPbuildrXrQrRrSro input_shapes r4rzTFGPTJAttention.build s ::  44 ( 4t{{//0 @ !!4t~~">? @ 44 ( 4t{{//0 @ !!4t~~">? @ 44 ( 4t{{//0 @ !!4t~~">? @ 4T * 6t}}112 B ##T4$@A B B 7 @ @ @ @ @ @ B Bs0 )F32)G)G )G3F=G  GG!rpr)return tf.Tensor)r$ztf.DTyperr)rrrrvrrrrrrNN) rrrrrrrtf.Tensor | NonerrrzTuple[tf.Tensor, tf.Tensor]NNNNFF)rrrz%Optional[Tuple[tf.Tensor, tf.Tensor]]rrrrrrrrvrrvru) __name__ __module__ __qualname__rVry staticmethodr|rrrrr __classcell__rss@r4rJrJQs0]du11 q 4$,0&* #)#)#) #) ) #) $ #) %#)P=A+/)-&*"'= =:=) = ' = $ == =~Br6rJc0eZdZdfd ZddZddZxZS) TFGPTJMLPc t|di||j}tjj |t |jd|_tjj |t |jd|_ t|j|_ tjj|j|_|j|_||_y)Nfc_inrOrPfc_outrT)rUrVn_embdrr^rdrrerrractivation_functionactr_ embd_pdropdropoutrXintermediate_size)rorrprqrXrss r4rVzTFGPTJMLP.__init__s "6"MM \\'' /&BZBZ2[bi(  ll(( /&:R:R*SZb)  %V%?%?@||++F,=,=> !2r6c|j|}|j|}|j|}|j|}|Sru)rrrr)rors r4rzTFGPTJMLP.call-s@ =1 /  M2  ]3 r6c|jryd|_t|ddZtj|jj 5|jj dd|jgdddt|dd[tj|jj 5|jj dd|jgdddyy#1swYrxYw#1swYyxYw)NTrr) rrr'rrrPrrXrrrs r4rzTFGPTJMLP.build4s ::  4$ ' 3tzz/ ?   $dnn!=> ? 44 ( 4t{{//0 H !!4t/E/E"FG H H 5 ? ? H Hs )C%2)C1%C.1C:)rintrprrrurrrrVrrrrs@r4rrs3  Hr6rcXeZdZdfd Z d ddZddZxZS) TFGPTJBlockc *t|di||j |jnd|jz}tj j |jd|_t|d|_ t||d|_ ||_ y)Nr~ln_1epsilonrPattnrPmlprT)rUrVn_innerrrr^LayerNormalizationlayer_norm_epsilonrrJrrrrp)rorprq inner_dimrss r4rVzTFGPTJBlock.__init__Asw "6"&,nn&@FNNa&--FW LL33Fz'TFGPTJMainLayer.call..s"aQ2::a1G#H"as #c3&K|] }|| ywrurT)rvs r4rz'TFGPTJMainLayer.call..srqdedqrs)last_hidden_statepast_key_valuesr attentions)r[rr'r=rr expand_dimsr)r{r(r$multiplysubtractrrrrrr enumerateziprtupler )ro input_idsrrtoken_type_idsrr inputs_embedsrrrrrr past_lengthattention_mask_shapeone_csttoken_type_embedsr output_shapepresentsall_attentionsall_hidden_statesrblockrrrs @r4rzTFGPTJMainLayer.calls  ]%>cd d  "$Y/K 9r;r?.CDI  &$]3CR8KTU U  "K#fs466{2O$_Q%7%:;B?K  >>"((; BR]@]*^efgL  % $.n#= ZZ9Ma9PRSUVXlmnXo8pqNkk#&GWW^7==IN[[Wn)Mr{{[cOdeN  % %!7!77Izz,Z 5Mb5Q0RS  *9dhh6I6I J HHY[HAM  %ZZZ=WXZ=[8\]N $k J  " C 0 GG$5]=P=PQ%(99  -( C "j&?&C%DD "20d"6BD&/DFFO0L&M T "A"z#$5MS_9`8b$b!+%-)#A,#"3! G$AJM#wqzm3 !/7 1q3Q2S!S) T, -0  =,?  1]4D D  %0"%5%OQ_$`rr r(+$+%   r6c|jryd|_t|ddMtj|jj 5|jj ddddt|ddZtj|jj 5|jj dd|jgdddt|ddK|jD];}tj|j 5|j dddd=yy#1swYxYw#1swYnxYw#1swYaxYw)NTrrr) rrr'rrrPrrrXr)rorlayers r4rzTFGPTJMainLayer.build!s  ::  4 % 1txx}}- %t$ % 4 & 2tyy~~. > tT^^ <= > 4d # / &]]5::.&KK%&& & 0  % % > >&&s$ D/%)D;E/D8;EE r)rr NNNNNNNNNNNF)r2Union[TFBaseModelOutputWithPast, Tuple[tf.Tensor]]ru) rrrr config_classrVrrrrrrrrs@r4rrysiL',3" !~  <~ ~ @ &r6rceZdZdZeZdZdgZy)TFGPTJPreTrainedModelz An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained models. transformerh.\d+.attn.biasN)rrr__doc__rrbase_model_prefix"_keys_to_ignore_on_load_unexpectedrTr6r4rr1s L%*<)=&r6ra| This model inherits from [`TFPreTrainedModel`]. Check the superclass documentation for the generic methods the library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads etc.) This model is also a [keras.Model](https://www.tensorflow.org/api_docs/python/tf/keras/Model) subclass. Use it as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage and behavior. TensorFlow models and layers in `transformers` accept two formats as input: - having all inputs as keyword arguments (like PyTorch models), or - having all inputs as a list, tuple or dict in the first positional argument. The reason the second format is supported is that Keras methods prefer this format when passing inputs to models and layers. Because of this support, when using methods like `model.fit()` things should "just work" for you - just pass your inputs and labels in any format that `model.fit()` supports! If, however, you want to use the second format outside of Keras methods like `fit()` and `predict()`, such as when creating your own layers or models with the Keras `Functional` API, there are three possibilities you can use to gather all the input Tensors in the first positional argument: - a single Tensor with `input_ids` only and nothing else: `model(input_ids)` - a list of varying length with one or several input Tensors IN THE ORDER given in the docstring: `model([input_ids, attention_mask])` or `model([input_ids, attention_mask, token_type_ids])` - a dictionary with one or several input Tensors associated to the input names given in the docstring: `model({"input_ids": input_ids, "token_type_ids": token_type_ids})` Note that when creating models and layers with [subclassing](https://keras.io/guides/making_new_layers_and_models_via_subclassing/) then you don't need to worry about any of this, as you can just pass inputs like you would to any other Python function! Parameters: config ([`GPTJConfig`]): Model configuration class with all the parameters of the model. Initializing with a config file does not load the weights associated with the model, only the configuration. Check out the [`~TFPreTrainedModel.from_pretrained`] method to load the model weights. a Args: input_ids (`Numpy array` or `tf.Tensor` of shape `(batch_size, input_ids_length)`): `input_ids_length` = `sequence_length` if `past` is `None` else `past[0].shape[-2]` (`sequence_length` of input past key value states). Indices of input sequence tokens in the vocabulary. If `past` is used, only input IDs that do not have their past calculated should be passed as `input_ids`. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.__call__`] and [`PreTrainedTokenizer.encode`] for details. [What are input IDs?](../glossary#input-ids) past_key_values (`List[tf.Tensor]` of length `config.n_layers`): Contains pre-computed hidden-states (key and values in the attention blocks) as computed by the model (see `past` output below). Can be used to speed up sequential decoding. The token ids which have their past given to this model should not be passed as input ids as they have already been computed. attention_mask (`tf.Tensor` or `Numpy array` of shape `(batch_size, sequence_length)`, *optional*): Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. [What are attention masks?](../glossary#attention-mask) token_type_ids (`tf.Tensor` or `Numpy array` of shape `(batch_size, sequence_length)`, *optional*): Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`: - 0 corresponds to a *sentence A* token, - 1 corresponds to a *sentence B* token. [What are token type IDs?](../glossary#token-type-ids) position_ids (`tf.Tensor` or `Numpy array` of shape `(batch_size, sequence_length)`, *optional*): Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`. [What are position IDs?](../glossary#position-ids) head_mask (`Numpy array` or `tf.Tensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - 1 indicates the head is **not masked**, - 0 indicates the head is **masked**. inputs_embeds (`tf.Tensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert `input_ids` indices into associated vectors than the model's internal embedding lookup matrix. output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. This argument can be used only in eager mode, in graph mode the value in the config will be used instead. output_hidden_states (`bool`, *optional*): Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for more detail. This argument can be used only in eager mode, in graph mode the value in the config will be used instead. return_dict (`bool`, *optional*): Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. This argument can be used in eager mode, in graph mode the value will always be set to True. training (`bool`, *optional*, defaults to `False`): Whether or not to use the model in training mode (some modules like dropout modules have different behaviors between training and evaluation). z_The bare GPT-J Model transformer outputting raw hidden-states without any specific head on top.ceZdZfdZeeeeee e  d ddZ ddZ xZ S) TFGPTJModelcPt||g|i|t|d|_y)Nrr)rUrVrrrorprrqrss r4rVzTFGPTJModel.__init__s) 3&3F3*6 Fr6 checkpoint output_typerc @|j||||||||| | | |  } | S)a  use_cache (`bool`, *optional*, defaults to `True`): If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see `past`). Set to `False` during training, `True` during generation r rrr rrr rrrrr)r)ror rrr rrr rrrrrrs r4rzTFGPTJModel.callsD8""+))%'/!5##  r6c|jryd|_t|ddNtj|jj 5|jj ddddyy#1swYyxYw)NTr)rrr'rrrPrrs r4rzTFGPTJModel.buildsm ::  4 - 9t//445 -  &&t, - - : - -s A11A:r)r TFModelInputType | Noner4Optional[Tuple[Tuple[Union[np.ndarray, tf.Tensor]]]]rnp.ndarray | tf.Tensor | Noner r/rr/rr/r r/rOptional[bool]rr0rr0rr0rr0rrru)rrrrVrr GPTJ_INPUTS_DOCSTRINGr _CHECKPOINT_FOR_DOCr _CONFIG_FOR_DOCrrrrs@r4r$r$s G*+@A&-$.2PT8<8<6:377;$(,0/3&*#($*$N$6 $ 6 $ 4 $1$5$"$*$-$$$!$ <$ B$L-r6r$zK The GPT-J Model transformer with a language modeling head on top. ceZdZfdZdZdZd dZeee jde e e e d d dZd dZxZS) TFGPTJForCausalLMct||g|i|t|d|_tj j |jt|jd|_ ||_ y)Nrrlm_headr) rUrVrrrr^rdrrrer7rpr&s r4rVzTFGPTJForCausalLM.__init__sf 3&3F3*6 F||))   /&BZBZ2[bk*   r6c|jSrur7rs r4get_output_embeddingsz'TFGPTJForCausalLM.get_output_embeddingss ||r6c||_yrur9)ronew_embeddingss r4set_output_embeddingsz'TFGPTJForCausalLM.set_output_embeddingss % r6c |jdd}|r77>>.rT>RL!~~l1b5.A2F  ,(.",   r6batch_size, sequence_lengthr'cF|j|||||||| | | | |  }|d}|j|}d}|(|ddddf}|ddddf}|j||}| s|f|ddz}||f|zS|St|||j|j |j S)a labels (`np.ndarray` or `tf.Tensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100` are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]` r+rNr8rlosslogitsrrr)rr7hf_compute_lossr rrr)ror rrr rrr labelsrrrrrtransformer_outputsr lm_logitsrFshifted_logitsoutputs r4rzTFGPTJForCausalLM.calls<#..+))%'/!5#/  ,A. LL/   &q#2#v.NAqrE]F''?D\$7$;;F)-)9TGf$ Ev E'/??-;;*55   r6c|jryd|_t|ddMtj|jj 5|jj ddddt|ddetj|jj 5|jj dd|jjgdddyy#1swY|xYw#1swYyxYw)NTrr7) rrr'rrrPrr7rprrs r4rzTFGPTJForCausalLM.buildUs ::  4 - 9t//445 -  &&t, - 4D ) 5t||001 E ""D$ 0B0B#CD E E 6 - - E E C"%3C."C+.C7r NNNNNNNNNNNNF)r r-rr.rr/r r/rr/rr/r r/rIr/rr0rr0rr0rr0rr0rz1Union[TFCausalLMOutputWithPast, Tuple[tf.Tensor]]ru)rrrrVr:r=rBrr r1formatr r2r r3rrrrs@r4r5r5s& 2*+@+G+GHe+fg&,$.2PT8<8<6:377;04$(,0/3&*#(9 *9 N9 6 9 6 9 4 9 19 59 .9 "9 *9 -9 $9 !9  ;9 h9 v Er6r5a The GPT-J Model transformer with a sequence classification head on top (linear layer). [`GPTJForSequenceClassification`] uses the last token in order to do the classification, as other causal models (e.g. GPT, GPT-2, GPT-Neo) do. Since it does classification on the last token, it requires to know the position of the last token. If a `pad_token_id` is defined in the configuration, it finds the last token that is not a padding token in each row. If no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in each row of the batch). ceZdZgdZfdZeeejde e e e  d ddZ d dZxZS) TFGPTJForSequenceClassificationzh.\d+.attn.masked_biasrzlm_head.weightc t||g|i||j|_t|d|_t j j|jdt|jd|_ ||_ y)NrrFscorerM) rUrV num_labelsrrrr^rdrrerVrpr&s r4rVz(TFGPTJForSequenceClassification.__init__ssu 3&3F3 ++*6 F\\'' OO.v/G/GH (   r6rCr'c~|3|jj|jddk7r td|j |||||||| | | | |  }|d}|j |}t |}d}|jjd}n|tjtjtjj||jj|jddz }tj|dk\|tjt |d|jdz }tj||dd}n.d}tj!|j"j$d d}|htj&|s |d|d|f}|j)tj*|dgtj*|d|j,g}||n|}| s|f|ddz}||f|zS|St/|||j0|j2|j4 S) a labels (`np.ndarray` or `tf.Tensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). Nrrz=Cannot handle batch sizes > 1 if no padding token is defined.r+r8r%) batch_dimsr&z will not detect padding tokens in `inputs_embeds`. Results may be unexpected if using padding tokens in conjunction with `inputs_embeds.`rE)rp pad_token_idshaper[rrVrr'argmaxr(r;equalr$rrlogger warning_oncersr is_tensorrHr=rWrrrr)ror rrr rrr rIrrrrrrJrrG logits_shape in_logitssequence_lengthsrF pooled_logitsrMs r4rz$TFGPTJForSequenceClassification.callsX:  $++":":"ByWXGY]^G^\] ]"..+))%'/!5#/  ,A. M*!&)  ;; # # +! $IIbggbggmmIt{{?W?W&XZcZiZijqst!$&88$)$GGJy}57G7M7MNQRR$ IIf.>1STU #% ##~~../0^^  << 01"1|A#68H#HI '' 6B4(@"**YY[]a]l]lXmBnoD%.%:  #%(;AB(??F)-)9TGf$ Ev E1 /??-;;*55   r6c|jryd|_t|ddMtj|jj 5|jj ddddt|ddetj|jj 5|jj dd|jjgdddyy#1swY|xYw#1swYyxYw)NTrrV) rrr'rrrPrrVrprrs r4rz%TFGPTJForSequenceClassification.builds ::  4 - 9t//445 -  &&t, - 4$ ' 3tzz/ C   $dkk.@.@!AB C C 4 - - C CrOrP)r r-rr.rr/r r/rr/rr/r r/rIr/rr0rr0rr0rr0rr0rz;Union[TFSequenceClassifierOutputWithPast, Tuple[tf.Tensor]]ru)rrr_keys_to_ignore_on_load_missingrVrr r1rQr r2rr3rrrrs@r4rSrSas" 'i# *+@+G+GHe+fg&6$.2PT8<8<6:377;04$(,0/3&*#(R *R NR 6 R 6 R 4 R 1R 5R .R "R *R -R $R !R  ER hR h Cr6rSz The GPT-J Model transformer with a span classification head on top for extractive question-answering tasks like SQuAD (a linear layers on top of the hidden-states output to compute `span start logits` and `span end logits`). ceZdZgdZfdZeeejde e e e  d ddZ d dZxZS) TFGPTJForQuestionAnsweringrTct||g|i||j|_t|d|_t j j|jt|jd|_ ||_ y)Nrr qa_outputsr) rUrVrWrrrr^rdrrerjrpr&s r4rVz#TFGPTJForQuestionAnswering.__init__sq 3&3F3 ++*6 F,,,, OO@X@X0Y`l-  r6rCr'c |j|||||||| | | |  }|d}|j|}tj|dd\}}tj|d}tj|d}d}|| d|i}| |d<|j |||f}| s||f|ddz}||f|zS|St ||||j|j S) a start_positions (`np.ndarray` or `tf.Tensor` of shape `(batch_size,)`, *optional*): Labels for position (index) of the start of the labelled span for computing the token classification loss. Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence are not taken into account for computing the loss. end_positions (`np.ndarray` or `tf.Tensor` of shape `(batch_size,)`, *optional*): Labels for position (index) of the end of the labelled span for computing the token classification loss. Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence are not taken into account for computing the loss. ) r rrr rrr rrrrrr"r8r%Nstart_position end_position)rF start_logits end_logitsrr) rrjr'rsqueezerHrrr)ror rrr rrr start_positions end_positionsrrrrrJsequence_outputrGrnrorFrIrMs r4rzTFGPTJForQuestionAnswering.calls!D#..+))%'/!5#/  .a01#%88FAB#? jzz,R8 ZZ 4   &=+D&8F%2F> "''z0JKD"J/2Eab2IIF)-)9TGf$ Ev E-%!-;;*55   r6c|jryd|_t|ddMtj|jj 5|jj ddddt|ddetj|jj 5|jj dd|jjgdddyy#1swY|xYw#1swYyxYw)NTrrj) rrr'rrrPrrjrprWrs r4rz TFGPTJForQuestionAnswering.buildAs ::  4 - 9t//445 -  &&t, - 4t , 8t334 M%%tT4;;3J3J&KL M M 9 - - M MrOrP)r r-rr.rr/r r/rr/rr/r r/rqr/rrr/rr0rr0rr0rr0rz7Union[TFQuestionAnsweringModelOutput, Tuple[tf.Tensor]]ru)rrrrfrVrr r1rQr r2rr3rrrrs@r4rhrhs'i#*+@+G+GHe+fg&2$.2PT8<8<6:377;9=7;,0/3&*#(? *? N? 6 ? 6 ? 4 ? 1? 5? 7? 5? *? -? $? !?  A? h? B Mr6rh)r/rr0rrr)r>rrr)rDrrErrr)>r  __future__rtypingrrrrinp tensorflowr'activations_tfr file_utilsr r r modeling_tf_outputsr r rrmodeling_tf_utilsrrrrrrrrrrtf_utilsrrrutilsrconfiguration_gptjr get_loggerrr^r2r3r5rArHr^LayerrJrrrrGPTJ_START_DOCSTRINGr1r$r5rSrhrTr6r4rs"))/    SR*   H %+EHBell((HBV!H ""!HH6%%,,$$6%rt&ell((t&t&n >- >(T<~e8-'8- 8-v  sE-/KsE  sEl sC&;=YsCsCl ]M!68O]M]Mr6