sg#/@dZddlmZddlZddlmZddlmZmZm Z m Z ddl Z ddl ZddlmZddlmZmZmZmZmZmZmZmZmZmZmZmZmZdd lm Z m!Z!m"Z"dd l#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)d d l*m+Z+e(jXe-Z.d Z/dZ0GddejbjdZ3GddejbjdZ4GddejbjdZ5GddejbjdZ6eGddejbjdZ7GddeZ8eGdde$Z9eGdde$Z:eGdd e$Z;eGd!d"e$Zd'Z?d(Z@e&d)e?Gd*d+e8ZAe&d,e?Gd-d.e8eZBe&d/e?Gd0d1e8eZCe&d2e?Gd3d4e8eZDe&d5e?Gd6d7e8eZEe&d8e?Gd9d:e8eZFy);z TF 2.0 XLNet model. ) annotationsN) dataclass)ListOptionalTupleUnion)get_tf_activation) TFCausalLanguageModelingLossTFModelInputTypeTFMultipleChoiceLossTFPreTrainedModelTFQuestionAnsweringLossTFSequenceClassificationLossTFSequenceSummaryTFSharedEmbeddingsTFTokenClassificationLossget_initializerkeraskeras_serializable unpack_inputs)check_embeddings_within_bounds shape_liststable_softmax) ModelOutputadd_code_sample_docstringsadd_start_docstrings%add_start_docstrings_to_model_forwardloggingreplace_return_docstrings) XLNetConfigzxlnet/xlnet-base-casedr"cleZdZfdZddZdZd dZ d dZd dZ d d dZ xZ S)TFXLNetRelativeAttentionc Ht|di||j|jzdk7r%t d|jd|j|j|_|j |_|j|_d|j dzz |_|j|_|j|_tjj|jd|_ tjj|j|_||_y) NrzThe hidden size (z6) is not a multiple of the number of attention heads (r!g? layer_normepsilonname)super__init__d_modeln_head ValueErrord_headscaleinitializer_rangeoutput_attentionsrlayersLayerNormalizationlayer_norm_epsr&Dropoutdropoutconfigselfr9kwargs __class__s ^/var/www/html/venv/lib/python3.12/site-packages/transformers/models/xlnet/modeling_tf_xlnet.pyr,z!TFXLNetRelativeAttention.__init__@s "6" >>FMM )Q .#FNN#34 --*  mm mm ~~ &--,- !'!9!9!'!9!9,,99&BWBW^j9k||++FNN;  c"t|j}|j|j|j|j f|dd|_|j|j|j|j f|dd|_|j|j|j|j f|dd|_|j|j|j|j f|dd|_ |j|j|j|j f|dd|_ |j|j|j fddd |_ |j|j|j fddd |_ |j|j|j fddd |_ |jd |j|j f|dd |_|jryd|_t!|ddet#j$|j&j(5|j&j+dd|j,jgdddyy#1swYyxYw)NTqshape initializer trainabler)kvorzerosr_r_biasr_s_biasr_w_bias seg_embedr&)rr2 add_weightr-r.r0rArFrGrHrIrKrLrMrObuiltgetattrtf name_scoper&r)buildr9)r; input_shaperDs r>rUzTFXLNetRelativeAttention.buildTsS%d&<&<= <<dkk: _cjm! <<dkk: _cjm! <<dkk: _cjm! <<dkk: _cjm! <<dkk: _cjm! ;; ,'TXb(  ;; ,'TXb(  ;; ,'TXb(  dkk4;;/[TX_j)  ::  4t , 8t334 I%%tT4;;3F3F&GH I I 9 I Is 3JJctNNotImplementedError)r;headss r> prune_headsz$TFXLNetRelativeAttention.prune_headsy!!r?ct|}tj||d|d|d|df}|dddf}tj||d|ddz |d|df}|ddd|ddddf}|S)z rel_shiftz"TFXLNetRelativeAttention.rel_shift|sA JJq6!9fQiF1IF G ab#gJ JJq6!9fQi!mVAYq J K a4Ao r?c tjd||jz|} tjd||jz|} |j | t | d} |d} nEtjd||j z|j} tjd|| } | | z| z|jz} |K|jtjk(s|jtjk(r | d|zz } n| d|zz } t| d }|j|| }|||z}tjd ||}|r||fS|S) z.Core relative positional attention operations.zibnd,jbnd->ijbnr!)rarzibnd,snd->ibnszijbs,ibns->ijbnigꌠ9Y>)Faxistrainingzijbn,jbnd->ibnd)rSeinsumrMrKrcrrLrOr1dtypefloat16bfloat16rr8)r;q_headk_head_hv_head_hk_head_rseg_mat attn_mask head_maskr3rhacbdef attn_score attn_probattn_vecs r> rel_attn_corez&TFXLNetRelativeAttention.rel_attn_coresM YY(&4==*@( KYY(&4==*@( K ^^BZ^A%6^ 7 ?B+Vdmm-CT^^TB,gr:B2gldjj0  "**, 2;;0N'%)*;; '$*:: #:A6 LLXL>   !I-I99. 8D Y& &r?ctjd||j}|j||}|r||z}|j |}|S)zPost-attention processing.z ibnd,hnd->ibhrg)rSrirHr8r&)r;hryresidualrhattn_outoutputs r>post_attentionz'TFXLNetRelativeAttention.post_attentionsK99_h?<<8<< !|H* r?c >||1tt|dkDrtj||gd} n|} tjd| |j } tjd| |j }tjd||j}tjd||j}|j|| ||||| | |  }| r|\}}|j||| }tjd||j}|Ptjd||}|j|| ||||| | |  }| r|\}}tjd||}n!|j|| ||||| | |  }| r|\}}|j||| }| rf}n|1tt|dkDrtj||gd} n|} tjd||j}tjd| |j } tjd| |j }tjd||j}|j|| ||||| | |  }| r|\}}|j||| }d}||f}| r|fz}|S)Nr!rrez ibh,hnd->ibndrgzmbnd,mlb->lbndzlbnd,mlb->mbnd) lenrrSconcatrirFrGrIrArzr)r;r|g attn_mask_h attn_mask_grIrqmemstarget_mappingrsr3rhcatrnrorpq_head_h attn_vec_h attn_prob_houtput_hq_head_g attn_vec_g attn_prob_goutput_grxryoutputss r>callzTFXLNetRelativeAttention.calls =C 4(8$9A$=iiq 2yy#tvv>Hyy#tvv>Hyy!TVVHyy#tvv>Hyy!TVVr$r$?sy(#IJ" pu*X */38<37,1I,I6I1I*IIr?r$c.eZdZfdZddZddZxZS)TFXLNetFeedForwardc t|di|tjj |j d|_tjj|jt|jd|_ tjj|jt|jd|_ tjj|j|_t!|j"t$r"t'|j"|_||_y|j"|_||_y)Nr&r'layer_1kernel_initializerr)layer_2r*)r+r,rr4r5r6r&Densed_innerrr2rr-rr7r8 isinstance ff_activationstrr activation_functionr9r:s r>r,zTFXLNetFeedForward.__init__Ns "6",,99&BWBW^j9k||)) NNv?W?W/X_h*  ||)) NNv?W?W/X_h*  ||++FNN; f**C 0'89M9M'ND $ (.';';D $ r?c|}|j|}|j|}|j||}|j|}|j||}|j ||z}|S)Nrg)rrr8rr&)r;inprhrs r>rzTFXLNetFeedForward.call^snf%))&1fx8f%fx8#. r?c|jryd|_t|dddtj|jj 5|jj dd|jjgdddt|dddtj|jj 5|jj dd|jjgdddt|ddetj|jj 5|jj dd|jjgdddyy#1swYxYw#1swYxYw#1swYyxYw)NTr&rr) rQrRrSrTr&r)rUr9r-rrrr;rVs r>rUzTFXLNetFeedForward.buildhsA ::  4t , 8t334 I%%tT4;;3F3F&GH I 4D ) 5t||001 F ""D$ 0C0C#DE F 4D ) 5t||001 F ""D$ 0C0C#DE F F 6  I I F F F Fs$ 3E*<3E6-3F*E36E?F rrXrrrr,rrUrrs@r>rrMs  Fr?rcLeZdZfdZ d ddZddZxZS) TFXLNetLayerc t|di|t|d|_t |d|_t jj|j|_ y)Nrel_attnr)ffr*) r+r,r$rrrrr4r7r8r:s r>r,zTFXLNetLayer.__init__xsI "6"0jI $V$7||++FNN; r?c |j||||||||| | |  } | dd\}}||j|| }|j|| }||f| ddz} | S)NrgrN)rr) r;rr non_tgt_maskrrpos_embrqrrrsr3rhrs r>rzTFXLNetLayer.call~s--            %Ra[(  wwx(w;H778h77X&4r?c|jryd|_t|ddMtj|jj 5|jj ddddt|ddNtj|jj 5|jj ddddyy#1swYexYw#1swYyxYw)NTrr)rQrRrSrTrr)rUrrs r>rUzTFXLNetLayer.builds ::  4T * 6t}}112 * ##D) * 4t $ 0tww||, $ d# $ $ 1 * * $ $ C %C CC rrrXrrs@r>rrwsV</38<37,1","6"1"*""H $r?rcFeZdZfdZfdZdZdZdZdZdZ xZ S) TFXLNetLMHeadc @t|di|||_||_y)Nr*)r+r,r9input_embeddings)r;r9rr<r=s r>r,zTFXLNetLMHead.__init__s$ "6" !1r?c|j|jjfddd|_t||y)NrJTbiasrB)rPr9 vocab_sizerr+rU)r;rVr=s r>rUzTFXLNetLMHead.builds7OO4;;+A+A*CQXdhouOv   k"r?c|jSrX)rr;s r>get_output_embeddingsz#TFXLNetLMHead.get_output_embeddingss$$$r?c`||j_t|d|j_yNr)rweightrrr;values r>set_output_embeddingsz#TFXLNetLMHead.set_output_embeddingss(',$+5e+get_biaszTFXLNetLMHead.get_biass ""r?cX|d|_t|dd|j_y)Nrr)rrr9rrs r>set_biaszTFXLNetLMHead.set_biass'&M !+E&M!:1!= r?cJ|j|d}||jz}|S)Nlinear)mode)rr)r; hidden_statess r>rzTFXLNetLMHead.calls+--m(-K % 1 r?) rrrr,rUrrrrrrrs@r>rrs'1#%@#>r?rceZdZeZfdZdZdZd dZdZ dZ dZ e d dZ d d Ze d d d ZxZS)TFXLNetMainLayerc ft|di|||_|j|_|j|_|j |_|j |_|j|_|j|_|j|_ |j|_ |j|_ |j|_ |j|_ |j|_|j|_t!|j"|j|jd|_t'|jDcgc]}t)|d|c}|_t,j.j1|j2|_|j4|_|j6|_ycc}w)Nword_embeddingr2r)zlayer_._rr*)r+r,r9output_hidden_statesr3 return_dictmem_len reuse_lenr- same_length attn_typebi_data clamp_lenn_layer use_bfloat16r2rrrrangerlayerrr4r7r8 use_mems_evaluse_mems_train)r;r9r<ir=s r>r,zTFXLNetMainLayer.__init__sP "6" $*$?$?!!'!9!9!--~~ ))~~ !--))~~ ))~~ "//!'!9!90   v~~AYAY`p JOv~~I^_Al6(1#?_ ||++FNN; #11$33 `s>F.c|jSrX)rrs r>get_input_embeddingsz%TFXLNetMainLayer.get_input_embeddingss"""r?c`||j_t|d|j_yr)rrrrrs r>set_input_embeddingsz%TFXLNetMainLayer.set_input_embeddingss(%*")3E):1)=&r?c8t|j}|jdd|jf|dd|_|j ryd|_t |ddMtj|jj5|jjddddt |ddK|jD];}tj|j5|jdddd=yy#1swYbxYw#1swYUxYw)Nr!Tmask_embrBrr) rr2rPr-rrQrRrSrTrr)rUr)r;rVrDrs r>rUzTFXLNetMainLayer.builds%d&<&<= a&K4V`(   ::  4)4 0 <t22778 0##))$/ 0 4$ ' 3 &]]5::.&KK%&& & 4 0 0&&sD&DD D ctrXrY)r;heads_to_prunes r> _prune_headszTFXLNetMainLayer._prune_headsr]r?ctj||g}tjj|dd}tjj|dd}tj||g}tj |||z gd}|j rQtjj|dd}tj |ddd|f|z|z |dd|dfgd}|S)a Creates causal attention mask. Float mask where 1.0 indicates masked, 0.0 indicates not-masked. Args: qlen: TODO Lysandre didn't fill mlen: TODO Lysandre didn't fill ``` same_length=False: same_length=True: < qlen > < qlen > ^ [0 0 0 0 0 1 1 1 1] [0 0 0 0 0 1 1 1 1] [0 0 0 0 0 0 1 1 1] [1 0 0 0 0 0 1 1 1] qlen [0 0 0 0 0 0 0 1 1] [1 1 0 0 0 0 0 1 1] [0 0 0 0 0 0 0 0 1] [1 1 1 0 0 0 0 0 1] v [0 0 0 0 0 0 0 0 0] [1 1 1 1 0 0 0 0 0] ``` rrr!N)rSoneslinalg band_partrJrr) r;qlenmlenrrmask_umask_dia attn_mask_padretmask_ls r> create_maskzTFXLNetMainLayer.create_mask s&GGT4L) $$Y2699&&y!Q7$. ii(9:A>   YY((B:F))SETE]V3h>AtuH NPQRC r?c|j|jdkDr|d|j}|j|jdk(rd}n |j }|||d}ntj||gd|d}tj|Sr)rrrSr stop_gradient)r;curr_outprev_memcutoffnew_mems r> cache_memzTFXLNetMainLayer.cache_mem&s >> %$..1*< 0$..1H << 4<<1#4Fll]F  vw'Gii8 4a8AG((r?ctjd||}tjtj|tj|gd}|dddddf}|tj |d|dg}|S)Nzi,d->idrrer!)rSrirsincostile)pos_seqinv_freqbsz sinusoid_inprs r>positional_embeddingz%TFXLNetMainLayer.positional_embedding;skyyGX> ))RVVL1266,3GHrR!T1*% ?ggg3{3Gr?ctjd|jd}dd||jz zz }|jdk(r|| }}n-|jdk(r|d}}nt d|jd |j rtj||d }tj| | d } |j dkDrXtj||j |j }tj| |j |j } |D|d zdk7rt d |d|j|||d z} |j| ||d z} n$|j||} |j| |} tj| | gd} | Stj||d }|j dkDr,tj||j |j }|j|||} | S)z$create relative positional encoding.rg@r!i'biunirzUnknown `attn_type` .g?rNzWith bi_data, the batch size z should be divisible by 2re) rSrr-rr/rr clip_by_valuer r) r;rrar freq_seqr begend fwd_pos_seq bwd_pos_seq fwd_pos_emb bwd_pos_embrs r>relative_positional_encodingz-TFXLNetMainLayer.relative_positional_encodingFs88At||S1(T\\"9:; >>T !dUC ^^u $RC3DNN3C1EF F <<((3T2K((C4#s3K~~! ..{T^^OT^^\  ..{T^^OT^^\ 7a<$'DSEIb%cdd"77 XsVWxX "77 XsVWxX "77 XN "77 XN iik :CG ((3T2K~~! ..{T^^OT^^\ // XsKGr?c |r| |j} n |j} | | td|)tj|d}t |dd\}}n6| )tj| d} t | dd\}}n td|tj|dnd}|tj|dnd}|tj|dnd}|tj|dnd}|tj|dnd}||dt |ddnd}||z}|j d k(r |j||}|ddddddf}n*|j d k(rd}ntd |j | |Jd |;|9tjd }d tj||jz }| | |d|z}n|||d}n |||}nd}|f|dkDr=tjt |d||g}tj||gd}||dddddddf}n||dddddddfz }|$tj|dkD|j}||tj| }|dkDr.tjtj||g|gd}tj||ddddddfzdkD|j}nd}| | }n1t||jj |j|}|j#||}|Ctj$|j&t |d|dg}|j#||}nd}||dkDr z(TFXLNetMainLayer.call..(s-%j"gi%jbcbll19&E&E%j&E%js(*c3JK|]}tj|dywr"r#)r%r&s r>r'z(TFXLNetMainLayer.call..*s%_2bll2I&F&F%_!#c3@K|]}td|Dyw)c3JK|]}tj|dyw)rNr rr!rNr#)r% attn_streams r>r'z2TFXLNetMainLayer.call.../s\;",,{FF\r)N)tupler%ts r>r'z(TFXLNetMainLayer.call...s!#abE\Z[\\#sc3JK|]}tj|dywr,r#r/s r>r'z(TFXLNetMainLayer.call..2s"Z!2<< #E#E"Zr)c3&K|] }|| ywrXr*)r%rGs r>r'z(TFXLNetMainLayer.call..5scqUVUbcs)last_hidden_staterr attentions)!rrr/rSr$rrrconstantcastrjrJreyerrrr8rr logical_notequalone_hotrrZrrr enumeraterappendr.TFXLNetModelOutput)'r; input_idsattention_maskr perm_maskrtoken_type_ids input_maskrs inputs_embedsuse_memsr3rrrhrr rrarrone_cst data_mask mems_maskr word_emb_kr word_emb_qrmem_padcat_idsrqrnew_memsr4rr layer_modulerrs' r>rzTFXLNetMainLayer.callnsp$ (**H))H  ]%>cd d  " YVH>TR\\*6:Z^ FTF`n6Bfj?H?TBLL;Z^ IWIcn9Eim)-)9d1g>Qz$q'"1%WXd{ >>U "((t4I!!Qd"23I ^^t #I;DNN;KLM M!^%;  l ;  ."<kk#&Grww~W]]KKJ  !i&;"4(94I  # (9"4(I  I$9!II  axHHj&;A&>c%JK IIy)&<1E  %aAtm4 Yq!Q}55   A Y__EI  FF4L=Lax!yy"((D$<*@,)OVXY 77I Q4=M0N$NRS#S[g[m[mnLL  $&J *9d6I6I6T6T U,,Y7J<< X<>  %N1KA1NPSUV0WXJ||J|BHH  %ax((D#;n6J6JK))Wn$=qA(ggrxxq$w(?qAQRS$**Gjj!,GG33D$C3H,,w,:  % %-I <6C O+D,R$ 2 (4 .OA|#t~~hQ'H&JJ#$$X=Qh%9W_`"Q! !! G")! Hh !!'!*-- .2  9M(H!5S[ \(*>hHW_`f95H # %%jP]%j j %%_Q^%_ _ )"#fp# #"Zz"ZZ cVX}j$Qcc c!$8=]g  r?rXNNNNNNNNNNNNNF)r>TFModelInputType | Noner?rrrr@rrrrArrBrrsrrCrrDrr3rrrrrrhr)rrrr" config_classr,rrrUrrr staticmethodr rrrrrs@r>rrsL48#>&"":)*&P.28<.2378<8<48377;#',0/3&*J *J 6J , J 1 J 6 J 6J 2J 1J 5J !J *J -J $J J J r?rceZdZdZeZdZy)TFXLNetPreTrainedModelz An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained models. transformerN)rrr__doc__r"rPbase_model_prefixr*r?r>rSrS<s L%r?rScJeZdZUdZdZded<dZded<dZded<dZded <y) r=a Output type of [`TFXLNetModel`]. Args: last_hidden_state (`tf.Tensor` of shape `(batch_size, num_predict, hidden_size)`): Sequence of hidden-states at the last layer of the model. `num_predict` corresponds to `target_mapping.shape[1]`. If `target_mapping` is `None`, then `num_predict` corresponds to `sequence_length`. mems (`List[tf.Tensor]` of length `config.n_layers`): Contains pre-computed hidden-states. Can be used (see `mems` input) to speed up sequential decoding. The token ids which have their past given to this model should not be passed as `input_ids` as they have already been computed. hidden_states (`tuple(tf.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`. Hidden-states of the model at the output of each layer plus the initial embedding outputs. attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`): Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`. Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. N tf.Tensorr3List[tf.Tensor] | NonerTuple[tf.Tensor, ...] | Nonerr4) rrrrUr3__annotations__rrr4r*r?r>r=r=Fs54$(y'#'D '26M/6/3J,3r?r=cXeZdZUdZdZded<dZded<dZded<dZd ed <dZ d ed <y) TFXLNetLMHeadModelOutputa Output type of [`TFXLNetLMHeadModel`]. Args: loss (`tf.Tensor` of shape *(1,)*, *optional*, returned when `labels` is provided) Language modeling loss (for next-token prediction). logits (`tf.Tensor` of shape `(batch_size, num_predict, config.vocab_size)`): Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). `num_predict` corresponds to `target_mapping.shape[1]`. If `target_mapping` is `None`, then `num_predict` corresponds to `sequence_length`. mems (`List[tf.Tensor]` of length `config.n_layers`): Contains pre-computed hidden-states. Can be used (see `mems` input) to speed up sequential decoding. The token ids which have their past given to this model should not be passed as `input_ids` as they have already been computed. hidden_states (`tuple(tf.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`. Hidden-states of the model at the output of each layer plus the initial embedding outputs. attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`): Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`. Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. Ntf.Tensor | NonelossrXlogitsrYrrZrr4 rrrrUr_r[r`rrr4r*r?r>r]r]hs?8"D !FI#'D '26M/6/3J,3r?r]cXeZdZUdZdZded<dZded<dZded<dZd ed <dZ d ed <y) &TFXLNetForSequenceClassificationOutputa) Output type of [`TFXLNetForSequenceClassification`]. Args: loss (`tf.Tensor` of shape `(1,)`, *optional*, returned when `label` is provided): Classification (or regression if config.num_labels==1) loss. logits (`tf.Tensor` of shape `(batch_size, config.num_labels)`): Classification (or regression if config.num_labels==1) scores (before SoftMax). mems (`List[tf.Tensor]` of length `config.n_layers`): Contains pre-computed hidden-states. Can be used (see `mems` input) to speed up sequential decoding. The token ids which have their past given to this model should not be passed as `input_ids` as they have already been computed. hidden_states (`tuple(tf.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`. Hidden-states of the model at the output of each layer plus the initial embedding outputs. attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`): Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`. Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. Nr^r_rXr`rYrrZrr4rar*r?r>rcrc?2"D !FI#'D '26M/6/3J,3r?rccXeZdZUdZdZded<dZded<dZded<dZd ed <dZ d ed <y) #TFXLNetForTokenClassificationOutputa Output type of [`TFXLNetForTokenClassificationOutput`]. Args: loss (`tf.Tensor` of shape `(1,)`, *optional*, returned when `labels` is provided) : Classification loss. logits (`tf.Tensor` of shape `(batch_size, sequence_length, config.num_labels)`): Classification scores (before SoftMax). mems (`List[tf.Tensor]` of length `config.n_layers`): Contains pre-computed hidden-states. Can be used (see `mems` input) to speed up sequential decoding. The token ids which have their past given to this model should not be passed as `input_ids` as they have already been computed. hidden_states (`tuple(tf.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`. Hidden-states of the model at the output of each layer plus the initial embedding outputs. attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`): Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`. Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. Nr^r_rXr`rYrrZrr4rar*r?r>rfrfrdr?rfcXeZdZUdZdZded<dZded<dZded<dZd ed <dZ d ed <y) TFXLNetForMultipleChoiceOutputa. Output type of [`TFXLNetForMultipleChoice`]. Args: loss (`tf.Tensor` of shape *(1,)*, *optional*, returned when `labels` is provided): Classification loss. logits (`tf.Tensor` of shape `(batch_size, num_choices)`): *num_choices* is the second dimension of the input tensors. (see *input_ids* above). Classification scores (before SoftMax). mems (`List[tf.Tensor]` of length `config.n_layers`): Contains pre-computed hidden-states. Can be used (see `mems` input) to speed up sequential decoding. The token ids which have their past given to this model should not be passed as `input_ids` as they have already been computed. hidden_states (`tuple(tf.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`. Hidden-states of the model at the output of each layer plus the initial embedding outputs. attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`): Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`. Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. Nr^r_rXr`rYrrZrr4rar*r?r>rhrhs?6"D !FI#'D '26M/6/3J,3r?rhcfeZdZUdZdZded<dZded<dZded<dZded <dZ d ed <dZ d ed <y) 'TFXLNetForQuestionAnsweringSimpleOutputa Output type of [`TFXLNetForQuestionAnsweringSimple`]. Args: loss (`tf.Tensor` of shape `(1,)`, *optional*, returned when `labels` is provided): Total span extraction loss is the sum of a Cross-Entropy for the start and end positions. start_logits (`tf.Tensor` of shape `(batch_size, sequence_length,)`): Span-start scores (before SoftMax). end_logits (`tf.Tensor` of shape `(batch_size, sequence_length,)`): Span-end scores (before SoftMax). mems (`List[tf.Tensor]` of length `config.n_layers`): Contains pre-computed hidden-states. Can be used (see `mems` input) to speed up sequential decoding. The token ids which have their past given to this model should not be passed as `input_ids` as they have already been computed. hidden_states (`tuple(tf.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`. Hidden-states of the model at the output of each layer plus the initial embedding outputs. attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`): Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`. Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. Nr^r_rX start_logits end_logitsrYrrZrr4) rrrrUr_r[rkrlrrr4r*r?r>rjrjsI6"D !"L)" J  #'D '26M/6/3J,3r?rja{ This model inherits from [`TFPreTrainedModel`]. Check the superclass documentation for the generic methods the library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads etc.) This model is also a [keras.Model](https://www.tensorflow.org/api_docs/python/tf/keras/Model) subclass. Use it as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage and behavior. TensorFlow models and layers in `transformers` accept two formats as input: - having all inputs as keyword arguments (like PyTorch models), or - having all inputs as a list, tuple or dict in the first positional argument. The reason the second format is supported is that Keras methods prefer this format when passing inputs to models and layers. Because of this support, when using methods like `model.fit()` things should "just work" for you - just pass your inputs and labels in any format that `model.fit()` supports! If, however, you want to use the second format outside of Keras methods like `fit()` and `predict()`, such as when creating your own layers or models with the Keras `Functional` API, there are three possibilities you can use to gather all the input Tensors in the first positional argument: - a single Tensor with `input_ids` only and nothing else: `model(input_ids)` - a list of varying length with one or several input Tensors IN THE ORDER given in the docstring: `model([input_ids, attention_mask])` or `model([input_ids, attention_mask, token_type_ids])` - a dictionary with one or several input Tensors associated to the input names given in the docstring: `model({"input_ids": input_ids, "token_type_ids": token_type_ids})` Note that when creating models and layers with [subclassing](https://keras.io/guides/making_new_layers_and_models_via_subclassing/) then you don't need to worry about any of this, as you can just pass inputs like you would to any other Python function! Parameters: config ([`XLNetConfig`]): Model configuration class with all the parameters of the model. Initializing with a config file does not load the weights associated with the model, only the configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. aP Args: input_ids (`torch.LongTensor` of shape `({0})`): Indices of input sequence tokens in the vocabulary. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*): Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. [What are attention masks?](../glossary#attention-mask) mems (`List[torch.FloatTensor]` of length `config.n_layers`): Contains pre-computed hidden-states (see `mems` output below) . Can be used to speed up sequential decoding. The token ids which have their past given to this model should not be passed as `input_ids` as they have already been computed. `use_mems` has to be set to `True` to make use of `mems`. perm_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length, sequence_length)`, *optional*): Mask to indicate the attention pattern for each input token with values selected in `[0, 1]`: - if `perm_mask[k, i, j] = 0`, i attend to j in batch k; - if `perm_mask[k, i, j] = 1`, i does not attend to j in batch k. If not set, each token attends to all the others (full bidirectional attention). Only used during pretraining (to define factorization order) or for sequential decoding (generation). target_mapping (`torch.FloatTensor` of shape `(batch_size, num_predict, sequence_length)`, *optional*): Mask to indicate the output tokens to use. If `target_mapping[k, i, j] = 1`, the i-th predict in batch k is on the j-th token. Only used during pretraining for partial prediction or for sequential decoding (generation). token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*): Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`: - 0 corresponds to a *sentence A* token, - 1 corresponds to a *sentence B* token. [What are token type IDs?](../glossary#token-type-ids) input_mask (`torch.FloatTensor` of shape `{0}`, *optional*): Mask to avoid performing attention on padding token indices. Negative of `attention_mask`, i.e. with 0 for real tokens and 1 for padding which is kept for compatibility with the original code base. Mask values selected in `[0, 1]`: - 1 for tokens that are **masked**, - 0 for tokens that are **not masked**. You can only uses one of `input_mask` and `attention_mask`. head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - 1 indicates the head is **not masked**, - 0 indicates the head is **masked**. inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert `input_ids` indices into associated vectors than the model's internal embedding lookup matrix. output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. output_hidden_states (`bool`, *optional*): Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for more detail. return_dict (`bool`, *optional*): Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. z_The bare XLNet Model transformer outputting raw hidden-states without any specific head on top.ceZdZfdZeeejdee e e  d ddZ ddZ xZS) TFXLNetModelcPt||g|i|t|d|_y)NrTr)r+r,rrTr;r9inputsr<r=s r>r,zTFXLNetModel.__init__s) 3&3F3+FGr?batch_size, sequence_length checkpoint output_typerPcD|j||||||||| | | | | |}|S)Nr>r?rr@rrArBrsrCrDr3rrrh)rT)r;r>r?rr@rrArBrsrCrDr3rrrhrs r>rzTFXLNetModel.callsJ0"")))!'/!5## "r?c|jryd|_t|ddNtj|jj 5|jj ddddyy#1swYyxYw)NTrT)rQrRrSrTrTr)rUrs r>rUzTFXLNetModel.buildsm ::  4 - 9t//445 -  &&t, - - : - -s A11A:rN)r>rOr?rrrr@rrrrArrBrrsrrCrrDrr3rrrrrrhrreturnz+Union[TFXLNetModelOutput, Tuple[tf.Tensor]]rX)rrrr,rrXLNET_INPUTS_DOCSTRINGformatr_CHECKPOINT_FOR_DOCr=_CONFIG_FOR_DOCrrUrrs@r>rnrns H*+A+H+HIf+gh&&$.28<.2378<8<48377;#',0/3&*"*"6", " 1 " 6 "6"2"1"5"!"*"-"$"" 5!" i"H-r?rnzt XLNet Model with a language modeling head on top (linear layer with weights tied to the input embeddings). ceZdZfdZdZdZd dZeee jde e e  d d dZd dZxZS) TFXLNetLMHeadModelct||g|i|t|d|_t ||jj d|_d|_y)NrTrlm_lossF)r+r,rrTrrrsupports_xla_generationrps r>r,zTFXLNetLMHeadModel.__init__sN 3&3F3+FG$VT-=-=-L-LS\] ',$r?c|jSrX)rrs r> get_lm_headzTFXLNetLMHeadModel.get_lm_heads ||r?ctjdt|jdz|jjzS)NzMThe method get_prefix_bias_name is deprecated. Please use `get_bias` instead./)warningswarn FutureWarningr)rrs r>get_prefix_bias_namez'TFXLNetLMHeadModel.get_prefix_bias_names/ egtuyy3!2!222r?c | |jd}tj|df|j}d |r$tj|dd df|gd}ntj||gd}|jd}tj|||dz f} tj ||df} tj| | gd} tj|d|dz f} tj |ddf} tj| | gd} || | |d}|rt fd|D|d <|S) Nrr!r rNrer)r>r@rrDc3<K|]}|d ddddfywrXr*)r% layer_pastoffsets r>r'zCTFXLNetLMHeadModel.prepare_inputs_for_generation..s""`*:hwh1n#="`sr)rCrSrJrjrrr.)r;rqpast_key_valuesrDr<effective_batch_size dummy_tokenr>sequence_lengthr@perm_mask_seq_endrtarget_mapping_seq_endrs @r>prepare_inputs_for_generationz0TFXLNetLMHeadModel.prepare_inputs_for_generationsD%||Ahh 4a8 M   6!fWX+#6 "D1MI 6;"7a@I$//!,HH2O_WXEXYZ GG%9?A$NOIIy*;<2F #7Oa1)E!FN4J#KRTU#",   ""`P_"``F6N r?rr)rurPc"|j||||||||| | | | | |}|d}|j||}d}||j||}| s|f|ddz}||f|zS|St|||j|j |j S)a labels (`tf.Tensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the cross entropy classification loss. Indices should be in `[0, ..., config.vocab_size - 1]`. Return: Examples: ```python >>> import tensorflow as tf >>> import numpy as np >>> from transformers import AutoTokenizer, TFXLNetLMHeadModel >>> tokenizer = AutoTokenizer.from_pretrained("xlnet/xlnet-large-cased") >>> model = TFXLNetLMHeadModel.from_pretrained("xlnet/xlnet-large-cased") >>> # We show how to setup inputs to predict a next token using a bi-directional context. >>> input_ids = tf.constant(tokenizer.encode("Hello, my dog is very ", add_special_tokens=True))[ ... None, : ... ] # We will predict the masked token >>> perm_mask = np.zeros((1, input_ids.shape[1], input_ids.shape[1])) >>> perm_mask[:, :, -1] = 1.0 # Previous tokens don't see last token >>> target_mapping = np.zeros( ... (1, 1, input_ids.shape[1]) ... ) # Shape [1, 1, seq_length] => let's predict one token >>> target_mapping[ ... 0, 0, -1 ... ] = 1.0 # Our first (and only) prediction will be the last token of the sequence (the masked token) >>> outputs = model( ... input_ids, ... perm_mask=tf.constant(perm_mask, dtype=tf.float32), ... target_mapping=tf.constant(target_mapping, dtype=tf.float32), ... ) >>> next_token_logits = outputs[ ... 0 ... ] # Output has shape [target_mapping.size(0), target_mapping.size(1), config.vocab_size] ```rwrrgNr!r_r`rrr4)rTrhf_compute_lossr]rrr4)r;r>r?rr@rrArBrsrCrDr3rrlabelsrhtransformer_outputs hidden_stater`r_rs r>rzTFXLNetLMHeadModel.calls~#..)))!'/!5#/  +1- lX>  ''7DY!4QR!88F)-)9TGf$ Ev E'$))-;;*55   r?c|jryd|_t|ddMtj|jj 5|jj ddddt|ddNtj|jj 5|jj ddddyy#1swYexYw#1swYyxYw)NTrTr)rQrRrSrTrTr)rUrrs r>rUzTFXLNetLMHeadModel.buildhs ::  4 - 9t//445 -  &&t, - 4D ) 5t||001 ) ""4( ) ) 6 - - ) )r)NNNNNNNNNNNNNNNNF) r>rOr?rrrr@rrrrArrBrrsrrCrrDrr3rrrrrrrrhrryz1Union[TFXLNetLMHeadModelOutput, Tuple[tf.Tensor]]rX)rrrr,rrrrrrzr{r r]r}rrUrrs@r>rrs>-3%N*+A+H+HIf+gh+CRab.28<.2378<8<48377;#',0/3&*04!] *] 6] , ] 1 ] 6 ] 6] 2] 1] 5] !] *] -] $] .] !] " ;#] ci] ~ )r?rz XLNet Model with a sequence classification/regression head on top (a linear layer on top of the pooled output) e.g. for GLUE tasks. ceZdZfdZeeejdee e e  d ddZ ddZ xZS) TFXLNetForSequenceClassificationcBt||g|i||j|_t|d|_t ||j d|_tjj|jt|j d|_ ||_ y)NrTrsequence_summaryr logits_projr)r+r, num_labelsrrTrr2rrr4rrrr9rps r>r,z)TFXLNetForSequenceClassification.__init__|s 3&3F3 +++FG 1 f&>&>EW! !<<--   /&BZBZ2[bo.  r?rrrsc@|j||||||||| | | | | |}|d}|j|}|j|}|dn|j||}| s|f|ddz}||f|zS|St |||j |j |jS)a labels (`tf.Tensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). rwrNr!r)rTrrrrcrrr4r;r>r?rr@rrArBrsrCrDr3rrrrhrrr`r_s r>rz%TFXLNetForSequenceClassification.calls>#..)))!'/!5#/  %Q'&&v.!!&)~t4+?+?+OY!4QR!88F)-)9TGf$ Ev E5$))-;;*55   r?c|jryd|_t|ddMtj|jj 5|jj ddddt|ddMtj|jj 5|jj ddddt|ddetj|jj 5|jj dd|jjgdddyy#1swYxYw#1swYxYw#1swYyxYwNTrTrr rQrRrSrTrTr)rUrrr9r-rs r>rUz&TFXLNetForSequenceClassification.build$ ::  4 - 9t//445 -  &&t, - 4+T 2 >t4499: 2%%++D1 2 4 - 9t//445 J  &&dDKK4G4G'HI J J :  - - 2 2 J J$ D<%E?3E<EEEr) r>rOr?rrrr@rrrrArrBrrsrrCrrDrr3rrrrrrrrhrryz?Union[TFXLNetForSequenceClassificationOutput, Tuple[tf.Tensor]]rX)rrrr,rrrzr{rr|rcr}rrUrrs@r>rrts" *+A+H+HIf+gh&:$.28<.2378<8<48377;#',0/3&*04!9 *9 69 , 9 1 9 6 9 69 29 19 59 !9 *9 -9 $9 .9 !9 " I#9 i9 v Jr?rz XLNET Model with a multiple choice classification head on top (a linear layer on top of the pooled output and a softmax) e.g. for RocStories/SWAG tasks. ceZdZfdZeeejdee e e  d ddZ ddZ xZS) TFXLNetForMultipleChoicec t||g|i|t|d|_t ||j d|_tjjdt|j d|_ ||_ y)NrTrrrr!rr) r+r,rrTrr2rrr4rrrr9rps r>r,z!TFXLNetForMultipleChoice.__init__s~ 3&3F3+FG 1 f&>&>EW! !<<-- /&2J2J"KR_.  r?z(batch_size, num_choices, sequence_lengthrsc|t|d}t|d}nt| d}t| d}|tj|d|fnd}|tj|d|fnd}|tj|d|fnd}|tj|d|fnd}| %tj| d|t| dfnd}|j|||||||||| | | | |}|d}|j |}|j |}tj|d|f}|dn|j ||}| s|f|ddz}||f|zS|St|||j|j|jS) a5 labels (`tf.Tensor` of shape `(batch_size,)`, *optional*): Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices]` where `num_choices` is the size of the second dimension of the input tensors. (See `input_ids` above) Nr!rNrr )rrhrr) rrSr_rTrrrrhrrr4)r;r>rArBr?rr@rrsrCrDr3rrrrh num_choices seq_lengthflat_input_idsflat_attention_maskflat_token_type_idsflat_input_maskflat_inputs_embedsrrr`reshaped_logitsr_s r>rzTFXLNetForMultipleChoice.calls>  $Y/2K#I.q1J$]3A6K#M215JDMDYIJ/?@_cN\Nhbjj"j9IJnrN\Nhbjj"j9IJnrFPF\"**Z"j1ABbf( JJ}r:z-7PQR7S&T U  #..            #/  %Q'&&v.!!&)**Vb+->?~t4+?+?+X%'*=ab*AAF)-)9TGf$ Ev E-"$))-;;*55   r?c|jryd|_t|ddMtj|jj 5|jj ddddt|ddMtj|jj 5|jj ddddt|ddetj|jj 5|jj dd|jjgdddyy#1swYxYw#1swYxYw#1swYyxYwrrrs r>rUzTFXLNetForMultipleChoice.build?rrr) r>rOrArrBrr?rrrr@rrrrsrrCrrDrr3rrrrrrrrhrryz7Union[TFXLNetForMultipleChoiceOutput, Tuple[tf.Tensor]]rX)rrrr,rrrzr{rr|rhr}rrUrrs@r>rrs4 *+A+H+HIs+tu&2$.28<488<.2378<377;#',0/3&*04!H *H 6H 2 H 6 H , H 1H 6H 1H 5H !H *H -H $H .H !H " A#H vH T Jr?rz XLNet Model with a token classification head on top (a linear layer on top of the hidden-states output) e.g. for Named-Entity-Recognition (NER) tasks. ceZdZfdZeeejdee e e  d ddZ ddZ xZS) TFXLNetForTokenClassificationct||g|i||j|_t|d|_t j j|jt|jd|_ ||_ y)NrTr classifierr) r+r,rrrTrr4rrr2rr9rps r>r,z&TFXLNetForTokenClassification.__init__Vss 3&3F3 +++FG,,,,   /&BZBZ2[bn-  r?rrrsc|j||||||||| | | | | |}|d}|j|}|dn|j||}| s|f|ddz}||f|zS|St|||j|j |j S)z labels (`tf.Tensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`. rwrNr!r)rTrrrfrrr4rs r>rz"TFXLNetForTokenClassification.call`s<#..)))!'/!5#/  %Q'(~t4+?+?+OY!4QR!88F)-)9TGf$ Ev E2$))-;;*55   r?c|jryd|_t|ddMtj|jj 5|jj ddddt|ddetj|jj 5|jj dd|jjgdddyy#1swY|xYw#1swYyxYw)NTrTr) rQrRrSrTrTr)rUrr9 hidden_sizers r>rUz#TFXLNetForTokenClassification.build ::  4 - 9t//445 -  &&t, - 4t , 8t334 M%%tT4;;3J3J&KL M M 9 - - M M C"%3C."C+.C7r) r>rOr?rrrr@rrrrArrBrrsrrCrrDrr3rrrrrrrrhrryzrrNs"*+A+H+HIf+gh&7$.28<.2378<8<48377;#',0/3&*04!5 *5 65 , 5 1 5 6 5 65 25 15 55 !5 *5 -5 $5 .5 !5 " F#5 i5 n Mr?rz XLNet Model with a span classification head on top for extractive question-answering tasks like SQuAD (a linear layers on top of the hidden-states output to compute `span start logits` and `span end logits`). ceZdZfdZeeejdee e e  d ddZ ddZ xZS) !TFXLNetForQuestionAnsweringSimplect||g|i|t|d|_tj j |jt|jd|_ ||_ y)NrTr qa_outputsr) r+r,rrTrr4rrrr2rr9rps r>r,z*TFXLNetForQuestionAnsweringSimple.__init__sf 3&3F3+FG,,,,   /&BZBZ2[bn-  r?rrrsc|j||||||||| | | | | |}|d}|j|}tj|dd\}}tj|d}tj|d}d}||d|i}||d<|j |||f}| s||f|d dz}||f|zS|St ||||j|j|j S) a start_positions (`tf.Tensor` of shape `(batch_size,)`, *optional*): Labels for position (index) of the start of the labelled span for computing the token classification loss. Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence are not taken into account for computing the loss. end_positions (`tf.Tensor` of shape `(batch_size,)`, *optional*): Labels for position (index) of the end of the labelled span for computing the token classification loss. Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence are not taken into account for computing the loss. rwrrNrreNstart_position end_positionr!)r_rkrlrrr4) rTrrSsplitsqueezerrjrrr4)r;r>r?rr@rrArBrsrCrDr3rrstart_positions end_positionsrhrsequence_outputr`rkrlr_rrs r>rz&TFXLNetForQuestionAnsweringSimple.calls3H#..)))!'/!5#/  .a01#%88FAB#? jzz,R8 ZZ 4   &=+D&8F%2F> "''z0JKD"J/2Eab2IIF)-)9TGf$ Ev E6%!$))-;;*55   r?c|jryd|_t|ddMtj|jj 5|jj ddddt|ddetj|jj 5|jj dd|jjgdddyy#1swY|xYw#1swYyxYw)NTrTr) rQrRrSrTrTr)rUrr9rrs r>rUz'TFXLNetForQuestionAnsweringSimple.buildrr)NNNNNNNNNNNNNNNF)"r>rOr?rrrr@rrrrArrBrrsrrCrrDrr3rrrrrrrrrrhrryz@Union[TFXLNetForQuestionAnsweringSimpleOutput, Tuple[tf.Tensor]]rX)rrrr,rrrzr{rr|rjr}rrUrrs@r>rrsB*+A+H+HIf+gh&;$.28<.2378<8<48377;#',0/3&*9=7;#E *E 6E , E 1 E 6 E 6E 2E 1E 5E !E *E -E $E 7E 5!E "#E $ J%E iE N Mr?r)GrU __future__rr dataclassesrtypingrrrrnumpynp tensorflowrSactivations_tfr modeling_tf_utilsr r r rrrrrrrrrrtf_utilsrrrutilsrrrrrr configuration_xlnetr" get_loggerrloggerr|r}r4Layerr$rrrrrSr=r]rcrfrhrjXLNET_START_DOCSTRINGrzrnrrrrrr*r?r>rs #!///SR-   H %.Ku||11K\'F++'FT4$5<<%%4$nELL&&@j u||))j j Z &.& 44 4B !4{!4 !4H 4[4 4B 4+4 4B  4[ 4  4F !4k!4 !4H(TFRe6-)6- 6-r  a)/1Ma)  a)H \J'=?[\J\J~ jJ57KjJjJZ RM$:@W`M`Mr?