sgdZddlZddlmZddlmZmZmZddlZddl Zddlm Z ddl m Z m Z mZddlmZdd lmZmZmZmZmZmZdd lmZdd lmZmZmZdd lmZm Z m!Z!m"Z"m#Z#m$Z$d dl%m&Z&e$jNe(Z)dZ*dZ+da,dZ-dZ.dZ/dZ0GddejbjdZ3GddejbjdZ4Gdde jjZ6Gdde jjZ7Gdde jjZ8Gdd e jjZ9Gd!d"e jjZ:Gd#d$e jjZ;Gd%d&e jjZ<Gd'd(e jjZ=Gd)d*e jjZ>Gd+d,e jjZ?Gd-d.e jjZ@Gd/d0eZAd1ZBd2ZCe d3eBGd4d5eAZDe d6eBGd7d8eAZEGd9d:e jjZFe d;eBGd<d=eAZGe d>eBGd?d@eAZHe dAeBGdBdCeAZIe dDeBGdEdFeAZJy)GzPyTorch YOSO model.N)Path)OptionalTupleUnion)nn)BCEWithLogitsLossCrossEntropyLossMSELoss)ACT2FN)"BaseModelOutputWithCrossAttentionsMaskedLMOutputMultipleChoiceModelOutputQuestionAnsweringModelOutputSequenceClassifierOutputTokenClassifierOutput)PreTrainedModel)apply_chunking_to_forward find_pruneable_heads_and_indicesprune_linear_layer)add_code_sample_docstringsadd_start_docstrings%add_start_docstrings_to_model_forwardis_ninja_availableis_torch_cuda_availablelogging) YosoConfigzuw-madison/yoso-4096rcHddlm}d}|gd}|d|dddlay)Nr)loadcttjjjjdz dz }|Dcgc]}||z  c}Scc}w)Nkernelsyoso)r__file__resolveparent)files src_folderfiles Y/var/www/html/venv/lib/python3.12/site-packages/transformers/models/yoso/modeling_yoso.py append_rootz&load_cuda_kernels..append_root=sJ(^++-44;;BBYNQWW .34d T!444s A)zfast_lsh_cumulation_torch.cppzfast_lsh_cumulation.cuzfast_lsh_cumulation_cuda.cufast_lsh_cumulationT)verbose)torch.utils.cpp_extensionr r,lsh_cumulation)r r+ src_filess r*load_cuda_kernelsr19s'.5vwI  480ct|trHsm-&# F'')**, JJv   **,)446Mr2ct|tr> Z^^- --r2c,eZdZedZedZy)YosoCumulationc N|d}dtjtj||jddtj z z |z}||dddddfz|dddddfz}tj||} |j ||||||||_| S)N hash_code_lenrrA)rMacosrP transposemathpisave_for_backwardconfig) ctx query_maskkey_maskrTrUvaluerjrc expectationcumulation_values r*forwardzYosoCumulation.forwardts/ 5::ell5#--B:O&PQTXT[T[[[`mm !Jq!Tz$::Xaqj=QQ  << U; j(KUS r2ct|}|j\}}}}}}|j}|d} tj||j dd|z} tj| | dz |z} tj| j dd| dz |z} tj|j dd|} dd| | | dfS)NrcrArdr@)r> saved_tensorsrjrMrPrf)rkgradrlrmrorTrUrnrjrc weighted_exp grad_querygrad_key grad_values r*backwardzYosoCumulation.backwardsT"?B?P?P< Hk5#u/ ||D%//"b*AB[P \\,1Bc0IJ << 6 6r2 >QRARV[@[\\\+"7"7B"?F T:xTAAr2N__name__ __module__ __qualname__ staticmethodrqryr2r*rarass*     B Br2rac,eZdZedZedZy)YosoLSHCumulationc |jd|jdk7r td|jd|jdk7r td|jd|jdk7r td|jd|jdk7r td|jd|jdk7r td|jd|jdk7r td t|||||g\}}}}}|j}|d }|d } t d| z} |d r t j |||||| |d\} } nt|||| \} } t j || || || |d} |j||| | |||||_ | S) Nrz6Query mask and Key mask differ in sizes in dimension 0z3Query mask and Query differ in sizes in dimension 0z1Query mask and Key differ in sizes in dimension 0z8Query mask and Value mask differ in sizes in dimension 0rz,Key and Value differ in sizes in dimension 1r@z,Query and Key differ in sizes in dimension 2rVrc use_fast_hash) rKrLr>is_cudarRr/ fast_hashr_rirj)rkrlrmrTrUrnrjuse_cudarVrchashtable_capacityquery_hash_code key_hash_coderps r*rqzYosoLSHCumulation.forwards ??1 q!1 1UV V ??1 A .RS S ??1 ! ,PQ Q ??1 A .WX X 88A;%**Q- 'KL L ::a=CHHQK 'KL L2?XW\^ach@i2j/ HeS%%%*%/  M!12 / "-;-E-EE8S(M8UV. *O].5UC=-Y *O])88 =%I[]egh  j(O]TY[^`ef r2c t|}|j\}}}}}}}|j} |j} | d} t d| z} | drft j |||||| | d} t j ||||||| dz |z| | d }t j ||||||| dz |z| | d }ndtjtj||jddtjz z | z}||dddddfz|dddddfz}tj||jdd|z}tj|| dz |z}tj|jdd| dz |z}tj|jdd|} dd||| dfS)Nrcr@ lsh_backwardrrArd) r>rsrjrrRr/lsh_weighted_cumulationrMrerPrfrgrh)rkrtrlrmrrrTrUrnrjrrcrrxrvrwrorus r*ryzYosoLSHCumulation.backwardsT"RURcRcO Ho}eS%<</  M!12 . !'66-_dL^`hjkJ(??"c)" J&=="e+" Huzz%,,ucmmBPR>S*TUX\X_X___dqqK% 1a:(>>!TST*AUUK <<eoob".EFTLl]Q5F#4MNJ||L$:$:2r$B]UVEVZ_D_`Hk&;&;B&CTJJT:xTAAr2Nrzrr2r*rrs+# # J.B.Br2rc*eZdZdZfdZddZxZS)YosoEmbeddingszGConstruct the embeddings from word, position and token_type embeddings.ctt|tj|j|j |j |_tj|jdz|j |_ tj|j|j |_ tj|j |j|_ tj|j|_|j#dt%j&|jj)ddzdt+|dd |_|j#d t%j.|j0j3t$j4|j0j6 dy) N) padding_idxr@eps position_ids)rrAF) persistentposition_embedding_typeabsolutetoken_type_idsdtyperH)super__init__r Embedding vocab_size hidden_size pad_token_idword_embeddingsmax_position_embeddingsposition_embeddingstype_vocab_sizetoken_type_embeddings LayerNormlayer_norm_epsDropouthidden_dropout_probdropoutregister_bufferrMrOexpandgetattrrzerosrrKlongrHselfrj __class__s r*rzYosoEmbeddings.__init__sL !||F,=,=v?Q?Q_e_r_rs#%<<0N0NQR0RTZTfTf#g %'\\&2H2H&J\J\%]"f&8&8f>S>STzz&"<"<=   ELL)G)GHOOPWX[\\in  (/v7PR\']$   KK))..0 4K\K\KcKc d  r2cT||j}n|jdd}|d}||jddd|f}|st|dr-|jddd|f}|j |d|}|}n:t j |t j|jj}||j|}|j|} || z} |jdk(r|j|} | | z } |j| } |j| } | S)NrArrrrr)rKrhasattrrrrMrrrHrrrrrr) r input_idsrr inputs_embeds input_shape seq_lengthbuffered_token_type_ids buffered_token_type_ids_expandedr embeddingsrs r*rqzYosoEmbeddings.forwards=  #..*K',,.s3K ^  ,,Q  ^ J"%::  ' ': 5"&":":<"H  - -J^^J/ \\*- r2)NNNNr{r|r}__doc__rrq __classcell__rs@r*rrsQ , r2rc.eZdZdfd ZdZddZxZS)YosoSelfAttentionct||j|jzdk7r2t |ds&t d|jd|jdt du}trtr |s t|j|_t|j|jz |_|j|jz|_t!j"|j|j|_t!j"|j|j|_t!j"|j|j|_t!j*|j,|_||n |j0|_|j2|_|j4|_|j6du|_|j:|_|j<|_|j>|_|j4|j:|j<|j>d|_ |j6Zt!jB|j|j|j6df|j6d zdfd |j |_"yy#t$r#}tjd|Yd}~Fd}~wwxYw) Nrembedding_sizezThe hidden size (z6) is not a multiple of the number of attention heads ()zGCould not load the custom kernel for multi-scale deformable attention: )rcrrVrrr@F) in_channels out_channels kernel_sizepaddingbiasgroups)#rrrnum_attention_headsrrLr/rrr1 ExceptionloggerwarningrRattention_head_size all_head_sizerLinearrTrUrnrattention_probs_dropout_probrruse_expectationrc conv_windowuse_convrrVr lsh_configConv2dconv)rrjr kernel_loadeders r*rzYosoSelfAttention.__init__)so     : : :a ?PVXhHi#F$6$6#78 445Q8 'd2 " $);)=m n!#$*#=#= #&v'9'9F'J #PVPnPn $ &55#11**$6 #11 "//"//!//  --      ) "66#77#//3++q0!411 DI *= n!hijhklmm ns= J&& K/K  Kc|jdd|j|jfz}|j|}|j ddddS)NrArr@rr )rKrrviewpermute)rlayernew_layer_shapes r*transpose_for_scoresz&YosoSelfAttention.transpose_for_scores\sO**,s+t/G/GIaIa.bb O,}}Q1a((r2cB|j|}|j|j|}|j|j|}|j|}|jr|j ||ddddddfz}|j \} } } } |j| | z| | }|j| | z| | }|j| | z| | }d|dz z}|jdj| dj| | z| j}d} |js| | kr| | z| | | z f}tj|tj||jgd}tj|tj||jgd}tj|tj||jgd}|js |j rt#||g\}}|jr%t$j'||||||j(}n$t*j'||||||j(}|js| | kr|ddddd| f}t#|}|j| | | | }|jr|z }|j-dd dd j/}|j dd |j0fz}|j2|}|r||f}|S|f}|S) N?g@rrI rGrArr@r rd)rTrrUrnrrrKrQ unsqueezerepeat_interleaverRrrMcatrrHtrainingrEraapplyrrrr8rr)r hidden_statesattention_maskoutput_attentionsmixed_query_layer key_layer value_layer query_layerconv_value_layer batch_size num_headsseq_lenhead_dim gpu_warp_sizepad_size context_layernew_context_layer_shapeoutputss r*rqzYosoSelfAttention.forwardas0 JJ}5--dhh}.EF // =0IJ //0AB ==#yy~aqRVFV7W)WX 3>3C3C3E0 Iw!))*y*@'8T %%j9&S"S* **,CD 4E=-0MZK[r2r4NF)r{r|r}rrrqrrs@r*rr(s1f) Qr2rcneZdZfdZdej dej dej fdZxZS)YosoSelfOutputc(t|tj|j|j|_tj |j|j|_tj|j|_ yNr) rrrrrdenserrrrrrs r*rzYosoSelfOutput.__init__s` YYv1163E3EF f&8&8f>S>STzz&"<"<= r2r input_tensorreturncr|j|}|j|}|j||z}|Sr4rrrrrrs r*rqzYosoSelfOutput.forward7 =1  ]3 }|'CD r2r{r|r}rrMTensorrqrrs@r*rr1> U\\RWR^R^r2rc.eZdZdfd ZdZddZxZS) YosoAttentionct|t|||_t ||_t |_y)N)r)rrrrroutputset pruned_heads)rrjrrs r*rzYosoAttention.__init__s3 %fF]^ $V, Er2c>t|dk(ryt||jj|jj|j \}}t |jj||j_t |jj||j_t |jj||j_ t |jj|d|j_ |jjt|z |j_|jj|jjz|j_ |j j||_y)NrrrI)rJrrrrrrrTrUrnr rrunion)rheadsindexs r* prune_headszYosoAttention.prune_headss u:? 7 49900$))2O2OQUQbQb u -TYY__eD *499==%@ ,TYY__eD .t{{/@/@%QO )- (E(EE (R %"&))"?"?$))B_B_"_  --33E:r2ch|j|||}|j|d|}|f|ddz}|S)Nrr)rr )rrrr self_outputsattention_outputrs r*rqzYosoAttention.forwardsCyy@QR ;;|A F#% QR(88r2r4r)r{r|r}rrrqrrs@r*r r s" ;$r2r cVeZdZfdZdej dej fdZxZS)YosoIntermediatect|tj|j|j |_t|jtrt|j|_ y|j|_ yr4) rrrrrintermediate_sizerr5 hidden_actstrr intermediate_act_fnrs r*rzYosoIntermediate.__init__s] YYv1163K3KL f'' -'-f.?.?'@D $'-'8'8D $r2rrcJ|j|}|j|}|Sr4)rrrrs r*rqzYosoIntermediate.forwards& =1 00? r2rrs@r*rrs#9U\\ellr2rcneZdZfdZdej dej dej fdZxZS) YosoOutputc(t|tj|j|j |_tj|j |j|_tj|j|_ yr) rrrrrrrrrrrrrs r*rzYosoOutput.__init__s` YYv779K9KL f&8&8f>S>STzz&"<"<= r2rrrcr|j|}|j|}|j||z}|Sr4rrs r*rqzYosoOutput.forwardrr2rrs@r*r!r!rr2r!c,eZdZfdZddZdZxZS) YosoLayerct||j|_d|_t ||_|j |_t||_t||_ yNr) rrchunk_size_feed_forward seq_len_dimr  attentionadd_cross_attentionr intermediater!r rs r*rzYosoLayer.__init__sW '-'E'E$&v.#)#=#= ,V4 ( r2c|j|||}|d}|dd}t|j|j|j|}|f|z}|S)N)rrr)r*rfeed_forward_chunkr(r))rrrrself_attention_outputsrr layer_outputs r*rqzYosoLayer.forward sh!% ~ar!s1!4(,0  # #T%A%A4CSCSUe  /G+r2cL|j|}|j||}|Sr4)r,r )rrintermediate_outputr0s r*r.zYosoLayer.feed_forward_chunks,"//0@A{{#68HI r2r)r{r|r}rrqr.rrs@r*r%r%s) r2r%c0eZdZfdZ ddZxZS) YosoEncoderct|||_tjt |j Dcgc] }t|c}|_d|_ ycc}wr) rrrjr ModuleListrangenum_hidden_layersr%rgradient_checkpointing)rrj_rs r*rzYosoEncoder.__init__!sN  ]]uVE]E]?^#_!If$5#_` &+#$`sA#cb|rdnd}|rdnd}t|jD]_\} } |r||fz}|jr+|jr|j | j |||} n | |||} | d}|sW|| dfz}a|r||fz}|st d|||fDSt|||S)Nrrrc3&K|] }|| ywr4r).0vs r* z&YosoEncoder.forward..Ismq_`_lms)last_hidden_stater attentions) enumeraterr9r_gradient_checkpointing_func__call__tupler ) rrr head_maskroutput_hidden_states return_dictall_hidden_statesall_self_attentionsi layer_module layer_outputss r*rqzYosoEncoder.forward's#7BD$5b4(4 POA|#$58H$H!**t}} $ A A ))!"% ! !-]NL] ^ )!,M &9]1=MS>STr2rrcl|j|}|j|}|j|}|Sr4)rrQrrs r*rqz#YosoPredictionHeadTransform.forward\s4 =1 --m< }5 r2rrs@r*rOrORs$UU\\ellr2rOc*eZdZfdZdZdZxZS)YosoLMPredictionHeadcHt|t||_t j |j |jd|_t jtj|j|_ |j|j_ y)NF)r) rrrO transformrrrrdecoder ParameterrMrrrs r*rzYosoLMPredictionHead.__init__esm 4V<yy!3!3V5F5FUS LLV->->!?@ !II r2c:|j|j_yr4)rrWrs r* _tie_weightsz!YosoLMPredictionHead._tie_weightsrs II r2cJ|j|}|j|}|Sr4)rVrWrs r*rqzYosoLMPredictionHead.forwardus$}5  ]3 r2)r{r|r}rr[rqrrs@r*rTrTds &&r2rTcVeZdZfdZdej dej fdZxZS)YosoOnlyMLMHeadcBt|t||_yr4)rrrT predictionsrs r*rzYosoOnlyMLMHead.__init__}s /7r2sequence_outputrc(|j|}|Sr4)r`)rraprediction_scoress r*rqzYosoOnlyMLMHead.forwards ,,_=  r2rrs@r*r^r^|s#8!u||! !r2r^c"eZdZdZeZdZdZdZy)YosoPreTrainedModelz An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained models. r#Tct|tjrm|jjj d|j j|j%|jjjyyt|tjrz|jjj d|j j|j2|jj|jjyyt|tjrJ|jjj|jjjdyy)zInitialize the weightsg)meanstdNr)r5rrweightdatanormal_rjinitializer_rangerzero_rrrfill_)rmodules r* _init_weightsz!YosoPreTrainedModel._init_weightss fbii ( MM   & &CT[[5R5R & S{{&   &&('  - MM   & &CT[[5R5R & S!!- ""6#5#56<<>.  - KK   " " $ MM   $ $S ).r2N) r{r|r}rr config_classbase_model_prefixsupports_gradient_checkpointingrprr2r*reres L&*#*r2reaG This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and behavior. Parameters: config ([`YosoConfig`]): Model configuration class with all the parameters of the model. Initializing with a config file does not load the weights associated with the model, only the configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. a5 Args: input_ids (`torch.LongTensor` of shape `({0})`): Indices of input sequence tokens in the vocabulary. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*): Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. [What are attention masks?](../glossary#attention-mask) token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*): Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`: - 0 corresponds to a *sentence A* token, - 1 corresponds to a *sentence B* token. [What are token type IDs?](../glossary#token-type-ids) position_ids (`torch.LongTensor` of shape `({0})`, *optional*): Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`. [What are position IDs?](../glossary#position-ids) head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - 1 indicates the head is **not masked**, - 0 indicates the head is **masked**. inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert *input_ids* indices into associated vectors than the model's internal embedding lookup matrix. output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. output_hidden_states (`bool`, *optional*): Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for more detail. return_dict (`bool`, *optional*): Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. z^The bare YOSO Model transformer outputting raw hidden-states without any specific head on top.c|eZdZfdZdZdZdZeejde e e e  ddeej deej d eej d eej d eej d eej d eedeedeedeee ffdZxZS) YosoModelct||||_t||_t ||_|jyr4)rrrjrrr4encoder post_initrs r*rzYosoModel.__init__s;   (0"6*  r2c.|jjSr4rrrZs r*get_input_embeddingszYosoModel.get_input_embeddingss...r2c&||j_yr4rz)rrns r*set_input_embeddingszYosoModel.set_input_embeddingss*/'r2c|jD]7\}}|jj|jj |9y)z Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base class PreTrainedModel N)itemsrwrr*r)rheads_to_prunerrs r* _prune_headszYosoModel._prune_headssE +002 CLE5 LL  u % / / ; ;E B Cr2batch_size, sequence_length checkpoint output_typerqrrrrrFrrrGrHrc ||n|jj}||n|jj}| | n|jj} | | t d|#|j |||j } n!||j dd} n t d| \} } | |jn |j} |tj| | f| }|pt|jdr4|jjddd| f}|j| | }|}n&tj| tj| }|j!||jj"}|j||||}|j%|||||| }|d }| s |f|d dzSt'||j(|j*|j, S) NzDYou cannot specify both input_ids and inputs_embeds at the same timerAz5You have to specify either input_ids or inputs_embedsrGrr)rrrr)rrFrrGrHrr)r@rrAcross_attentions)rjrrGuse_return_dictrL%warn_if_padding_and_no_attention_maskrKrHrMonesrrrrrr get_head_maskr8rwr rrAr)rrrrrrFrrrGrHrrrrHrrembedding_outputencoder_outputsras r*rqzYosoModel.forwards$2C1N-TXT_T_TqTq$8$D $++JjJj &1%cd d  "  6 6y. Q#..*K  &',,.s3KTU U!, J%.%:!! @T@T  !"ZZ*j)A6RN  !t(89*.//*H*HKZK*X'3J3Q3QR\^h3i0!A!&[ SY!Z&&y$++2O2OP ??%)' +  ,, )/!5# ' *!,#%(;; ;1-)77&11,==   r2) NNNNNNNNN)r{r|r}rr{r}rrYOSO_INPUTS_DOCSTRINGformatr_CHECKPOINT_FOR_DOCr _CONFIG_FOR_DOCrrMrboolrrrqrrs@r*rurus- /0C++@+G+GHe+fg&6$-11515/3,004,0/3&*I ELL)I !.I !. I u||, I ELL) I   -I $D>I 'tnI d^I  u88 9I h I r2ruz2YOSO Model with a `language modeling` head on top.ceZdZddgZfdZdZdZeejde e e e  ddeej d eej d eej d eej d eej d eej deej deedeedeedeee ffdZxZS)YosoForMaskedLMzcls.predictions.decoder.weightzcls.predictions.decoder.biasct||t||_t ||_|j yr4)rrrur#r^clsrxrs r*rzYosoForMaskedLM.__init__Qs4  f% "6* r2cB|jjjSr4)rr`rWrZs r*get_output_embeddingsz%YosoForMaskedLM.get_output_embeddingsZsxx##+++r2c||jj_|j|jj_yr4)rr`rWr)rnew_embeddingss r*set_output_embeddingsz%YosoForMaskedLM.set_output_embeddings]s,'5$$2$7$7!r2rrrrrrrFrlabelsrrGrHrc | | n|jj} |j|||||||| |  } | d} |j| } d}|Ft }|| j d|jj |j d}| s| f| ddz}||f|zS|St|| | j| jS)a labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. NrrrrFrrrGrHrrArlosslogitsrrA) rjrr#rr rrrrrA)rrrrrrFrrrrGrHrrarcmasked_lm_lossloss_fctr s r*rqzYosoForMaskedLM.forwardas2&1%2 'tn2 d^2  un$ %2 h 2 r2rc(eZdZdZfdZdZxZS)YosoClassificationHeadz-Head for sentence-level classification tasks.c4t|tj|j|j|_tj |j|_tj|j|j|_ ||_ yr4) rrrrrrrrr num_labelsout_projrjrs r*rzYosoClassificationHead.__init__sg YYv1163E3EF zz&"<"<=  &"4"4f6G6GH  r2c |dddddf}|j|}|j|}t|jj|}|j|}|j |}|S)Nr)rrr rjrr)rfeatureskwargsxs r*rqzYosoClassificationHead.forwardse Q1W  LLO JJqM 4;;)) *1 - LLO MM! r2rrs@r*rrs7r2rzYOSO Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled output) e.g. for GLUE tasks.ceZdZfdZeej deee e  dde e jde e jde e jde e jde e jd e e jd e e jd e ed e ed e edeee ffdZxZS)YosoForSequenceClassificationct|||j|_t||_t ||_|jyr4)rrrrur#r classifierrxrs r*rz&YosoForSequenceClassification.__init__sA   ++f% 08 r2rrrrrrrFrrrrGrHrc | | n|jj} |j|||||||| |  } | d} |j| } d}||jj|j dk(rd|j_nl|j dkDrL|j tjk(s|j tjk(rd|j_nd|j_|jjdk(rIt}|j dk(r&|| j|j}n|| |}n|jjdk(r=t}|| jd|j |jd}n,|jjdk(rt}|| |}| s| f| ddz}||f|zS|St|| | j | j" S) a labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). Nrrr regressionsingle_label_classificationmulti_label_classificationrAr)rjrr#r problem_typerrrMrrRr squeezer rrrrrA)rrrrrrFrrrrGrHrrarrrr s r*rqz%YosoForSequenceClassification.forwards2&1%C 'tnC d^C  u.. /C h C r2rzYOSO Model with a multiple choice classification head on top (a linear layer on top of the pooled output and a softmax) e.g. for RocStories/SWAG tasks.ceZdZfdZeej deee e  dde e jde e jde e jde e jde e jd e e jd e e jd e ed e ed e edeee ffdZxZS)YosoForMultipleChoicec t||t||_t j |j |j |_t j |j d|_|jyr') rrrur#rrrpre_classifierrrxrs r*rzYosoForMultipleChoice.__init__s_  f%  ii(:(:FG>SINN2y~~b'9:Y] M[Mg,,R1D1DR1HImqM[Mg,,R1D1DR1HImqGSG_|((\->->r-BCei (   r=#5#5b#9=;M;Mb;Q R  )) ))%'/!5#  qz $QT* ++M: ! -0 / ++b+6  ')HOV4D%''!"+5F)-)9TGf$ Ev E("!//))   r2r)r{r|r}rrrrrrrrrrMrrrrrqrrs@r*rr s4 ++@+G+GHr+st&-$-11515/3,004)-,0/3&*B ELL)B !.B !. B u||, B ELL) B   -B &B $D>B 'tnB d^B  u// 0B u B r2rzYOSO Model with a token classification head on top (a linear layer on top of the hidden-states output) e.g. for Named-Entity-Recognition (NER) tasks.ceZdZfdZeej deee e  dde e jde e jde e jde e jde e jd e e jd e e jd e ed e ed e edeee ffdZxZS)YosoForTokenClassificationc,t|||j|_t||_t j |j|_t j|j|j|_ |jyr4) rrrrur#rrrrrrrrxrs r*rz#YosoForTokenClassification.__init__msi   ++f% zz&"<"<= ))F$6$68I8IJ r2rrrrrrrFrrrrGrHrc | | n|jj} |j|||||||| |  } | d} |j| } |j | } d}|t }||j ddk(}| j d|j}tj||j dtj|jj|}|||}n2|| j d|j|j d}| s| f| ddz}||f|zS|St|| | j| jS)z labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`. NrrrArr)rjrr#rrr rrrMwherer= ignore_indextype_asrrrA)rrrrrrFrrrrGrHrrarrr active_loss active_logits active_labelsr s r*rqz"YosoForTokenClassification.forwardxsk.&1%; 'tn; d^;  u++ ,; h ; r2rzYOSO Model with a span classification head on top for extractive question-answering tasks like SQuAD (a linear layers on top of the hidden-states output to compute `span start logits` and `span end logits`).ceZdZfdZeej deee e  dde e jde e jde e jde e jde e jd e e jd e e jd e e jd e ed e ede edeee ffdZxZS)YosoForQuestionAnsweringct||d|_|j|_t||_t j |j|j|_|jy)Nr@) rrrrur#rrr qa_outputsrxrs r*rz!YosoForQuestionAnswering.__init__s[   ++f% ))F$6$68I8IJ r2rrrrrrrFrstart_positions end_positionsrrGrHrc | | n|jj} |j||||||| | |  } | d} |j| }|j dd\}}|j d}|j d}d}||t |jdkDr|j d}t |jdkDr|j d}|jd}|jd|}|jd|}t|}|||}|||}||zdz }| s||f| ddz}||f|zS|St|||| j| j S) a start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for position (index) of the start of the labelled span for computing the token classification loss. Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence are not taken into account for computing the loss. end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for position (index) of the end of the labelled span for computing the token classification loss. Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence are not taken into account for computing the loss. NrrrrArI)rr@)r start_logits end_logitsrrA) rjrr#rsplitrrJrKclampr rrrA)rrrrrrFrrrrrGrHrrarrr total_loss ignored_indexr start_lossend_lossr s r*rqz YosoForQuestionAnswering.forwards<&1%H 'tnH d^H  u22 3H h H r2r)KrrgpathlibrtypingrrrrMtorch.utils.checkpointrtorch.nnrr r activationsr modeling_outputsr rrrrrmodeling_utilsr pytorch_utilsrrrutilsrrrrrrconfiguration_yosor get_loggerr{rrrr/r1r>rEr_autogradFunctionrarModulerrrr rr!r%r4rOrTr^reYOSO_START_DOCSTRINGrrurrrrrrrr2r*rs )) AA!.ll+   H %, 1 C.&BU^^,,B>VB//VBt9RYY9xJ J\ RYY BIIB ryy    :. ")). d"))$2990!bii!*/*6 /ddh #h h VNPdeK )K fK \RYY*/ S $7S  S lH S /S  S lP M !4M  M `h [ 2[  [ r2