
    sg                        d Z ddlZddlmZ ddlmZmZmZ ddlZddl	Zddlm
Z
 ddlmZmZmZ ddlmZ dd	lmZmZmZmZmZmZ dd
lmZ ddlmZmZmZ ddlmZm Z m!Z!m"Z"m#Z#m$Z$ ddl%m&Z&  e$jN                  e(      Z)dZ*dZ+da,d Z-d Z.d Z/d Z0 G d dejb                  jd                        Z3 G d dejb                  jd                        Z4 G d de
jj                        Z6 G d de
jj                        Z7 G d de
jj                        Z8 G d d e
jj                        Z9 G d! d"e
jj                        Z: G d# d$e
jj                        Z; G d% d&e
jj                        Z< G d' d(e
jj                        Z= G d) d*e
jj                        Z> G d+ d,e
jj                        Z? G d- d.e
jj                        Z@ G d/ d0e      ZAd1ZBd2ZC e d3eB       G d4 d5eA             ZD e d6eB       G d7 d8eA             ZE G d9 d:e
jj                        ZF e d;eB       G d< d=eA             ZG e d>eB       G d? d@eA             ZH e dAeB       G dB dCeA             ZI e dDeB       G dE dFeA             ZJy)GzPyTorch YOSO model.    N)Path)OptionalTupleUnion)nn)BCEWithLogitsLossCrossEntropyLossMSELoss   )ACT2FN)"BaseModelOutputWithCrossAttentionsMaskedLMOutputMultipleChoiceModelOutputQuestionAnsweringModelOutputSequenceClassifierOutputTokenClassifierOutput)PreTrainedModel)apply_chunking_to_forward find_pruneable_heads_and_indicesprune_linear_layer)add_code_sample_docstringsadd_start_docstrings%add_start_docstrings_to_model_forwardis_ninja_availableis_torch_cuda_availablelogging   )
YosoConfigzuw-madison/yoso-4096r   c                  H    ddl m}  d } |g d      } | d|d       dd lay )Nr   )loadc                     t        t              j                         j                  j                  j                  dz  dz  }| D cg c]  }||z  	 c}S c c}w )Nkernelsyoso)r   __file__resolveparent)files
src_folderfiles      Y/var/www/html/venv/lib/python3.12/site-packages/transformers/models/yoso/modeling_yoso.pyappend_rootz&load_cuda_kernels.<locals>.append_root=   sJ    (^++-44;;BBYNQWW
.34d
T!444s   A)zfast_lsh_cumulation_torch.cppzfast_lsh_cumulation.cuzfast_lsh_cumulation_cuda.cufast_lsh_cumulationT)verbose)torch.utils.cpp_extensionr    r,   lsh_cumulation)r    r+   	src_filess      r*   load_cuda_kernelsr1   9   s'    .5 vwI		480    c                     t        | t              r<g }| D ]3  }|j                         s|j                         }|j	                  |       5 |S | j                         s| j                         } | S N)
isinstancelistis_contiguous
contiguousappendinput_tensorsouttensors      r*   to_contiguousr>   H   sm    -&# 	F'')**,JJv	 
**,)446Mr2   c                     t        | t              r<g }| D ]3  }|j                  t        j                  j                  |dd             5 |S t        j                  j                  | dd      S )N   )pdim)r5   r6   r9   r   
functional	normalizer:   s      r*   rE   rE   V   se    -&# 	EFJJr}}..v.CD	E
}}&&}r&BBr2   c                 z   t        | j                               dk7  rt        d      t        |j                               dk7  rt        d      t        j                  | j                  d      | j                  d      ||z  | j
                        }dt        j                  || j
                        z  }t        j                  | |      j                  | j                  d      | j                  d      ||      }t        j                  ||      j                  |j                  d      |j                  d      ||      }|dkD  j                         }|dkD  j                         }	t        j                  ||z  d	      }
t        j                  |	|z  d	      }
|
j                         |
j                         fS )
Nr   zQuery has incorrect size.zKey has incorrect size.r   r@   devicer   rA   rC   )lensize
ValueErrortorchrandnrH   arangematmulreshapeintsum)querykeynum_hashhash_lenrmat	raise_powquery_projectionkey_projectionquery_binary
key_binary
query_hashs              r*   hashingr_   `   sX   
5::<A455
388:!233;;uzz!}ejjmX5HQVQ]Q]^DU\\(5<<@@I||E4088A

STW_aij\\#t,44SXXa[#((1+xYabN$q(--/L 1$))+J<)3<J:	1r:J>>Z^^---r2   c                   ,    e Zd Zed        Zed        Zy)YosoCumulationc           
      N   |d   }dt        j                  t        j                  ||j                  dd                  t        j
                  z  z
  |z  }||d d d d d f   z  |d d d d d f   z  }t        j                  ||      }	| j                  ||||||       || _        |	S )Nhash_code_lenr   rA   )rM   acosrP   	transposemathpisave_for_backwardconfig)
ctx
query_maskkey_maskrT   rU   valuerj   rc   expectationcumulation_values
             r*   forwardzYosoCumulation.forwardt   s    /5::ell5#--B:O&PQTXT[T[[[`mm!Jq!Tz$::Xaqj=QQ <<U;j(KUS
r2   c                    t        |      }| j                  \  }}}}}}| j                  }|d   }	t        j                  ||j                  dd            |z  }
t        j                  |
|	dz  |z        }t        j                  |
j                  dd      |	dz  |z        }t        j                  |j                  dd      |      }d d |||d fS )Nrc   rA   rd   r@   )r>   saved_tensorsrj   rM   rP   rf   )rk   gradrl   rm   ro   rT   rU   rn   rj   rc   weighted_exp
grad_querygrad_key
grad_values                 r*   backwardzYosoCumulation.backward   s    T"?B?P?P<
Hk5#u/||D%//"b*AB[P\\,1Bc0IJ
<< 6 6r2 >QRARV[@[\\\+"7"7B"?F
T:xTAAr2   N__name__
__module____qualname__staticmethodrq   ry    r2   r*   ra   ra   s   s*    
  
  B Br2   ra   c                   ,    e Zd Zed        Zed        Zy)YosoLSHCumulationc           
         |j                  d      |j                  d      k7  rt        d      |j                  d      |j                  d      k7  rt        d      |j                  d      |j                  d      k7  rt        d      |j                  d      |j                  d      k7  rt        d      |j                  d      |j                  d      k7  rt        d      |j                  d      |j                  d      k7  rt        d	      t        |||||g      \  }}}}}|j                  }|d
   }|d   }	t	        d|	z        }
|d   r t
        j                  ||||||	|d      \  }}nt        ||||	      \  }}t
        j                  ||||||
|d      }| j                  |||||||       || _	        |S )Nr   z6Query mask and Key mask differ in sizes in dimension 0z3Query mask and Query differ in sizes in dimension 0z1Query mask and Key differ in sizes in dimension 0z8Query mask and Value mask differ in sizes in dimension 0r   z,Key and Value differ in sizes in dimension 1r@   z,Query and Key differ in sizes in dimension 2rV   rc   use_fast_hash)
rK   rL   r>   is_cudarR   r/   	fast_hashr_   ri   rj   )rk   rl   rm   rT   rU   rn   rj   use_cudarV   rc   hashtable_capacityquery_hash_codekey_hash_coderp   s                 r*   rq   zYosoLSHCumulation.forward   s   ??1q!11UVV??1A.RSS??1!,PQQ??1A.WXX88A;%**Q-'KLL::a=CHHQK'KLL2?XW\^ach@i2j/
HeS%%%*%/ M!12/"-;-E-EE8S(M8UV.*O] .5UC=-Y*O])88=%I[]egh
 	j(O]TY[^`ef
r2   c                    t        |      }| j                  \  }}}}}}}| j                  }	|j                  }
|	d   }t	        d|z        }|	d   rft
        j                  |||||||
d      }t
        j                  |||||||dz  |z  ||
d
      }t
        j                  |||||||dz  |z  ||
d
      }ndt        j                  t        j                  ||j                  dd                  t        j                  z  z
  |z  }||d d d d d f   z  |d d d d d f   z  }t        j                  ||j                  dd            |z  }t        j                  ||dz  |z        }t        j                  |j                  dd      |dz  |z        }t        j                  |j                  dd      |      }d d |||d fS )Nrc   r@   lsh_backwardr      rA   rd   )r>   rs   rj   r   rR   r/   lsh_weighted_cumulationrM   re   rP   rf   rg   rh   )rk   rt   rl   rm   r   r   rT   rU   rn   rj   r   rc   r   rx   rv   rw   ro   ru   s                     r*   ry   zYosoLSHCumulation.backward   s   T"RURcRcO
Ho}eS%<</ M!12.!'66-_dL^`hjkJ (??"c)"J &=="e+"H uzz%,,ucmmBPR>S*TUX\X_X___dqqK%
1a:(>>!TST*AUUK <<eoob".EFTLl]Q5F#4MNJ||L$:$:2r$B]UVEVZ_D_`Hk&;&;B&CTJJT:xTAAr2   Nrz   r   r2   r*   r   r      s+    #  # J .B .Br2   r   c                   *     e Zd ZdZ fdZddZ xZS )YosoEmbeddingszGConstruct the embeddings from word, position and token_type embeddings.c                 t   t         |           t        j                  |j                  |j
                  |j                        | _        t        j                  |j                  dz   |j
                        | _	        t        j                  |j                  |j
                        | _        t        j                  |j
                  |j                        | _        t        j                  |j                        | _        | j#                  dt%        j&                  |j                        j)                  d      dz   d       t+        |dd	      | _        | j#                  d
t%        j.                  | j0                  j3                         t$        j4                  | j0                  j6                        d       y )N)padding_idxr@   epsposition_ids)r   rA   F)
persistentposition_embedding_typeabsolutetoken_type_idsdtyperH   )super__init__r   	Embedding
vocab_sizehidden_sizepad_token_idword_embeddingsmax_position_embeddingsposition_embeddingstype_vocab_sizetoken_type_embeddings	LayerNormlayer_norm_epsDropouthidden_dropout_probdropoutregister_bufferrM   rO   expandgetattrr   zerosr   rK   longrH   selfrj   	__class__s     r*   r   zYosoEmbeddings.__init__   sL   !||F,=,=v?Q?Q_e_r_rs#%<<0N0NQR0RTZTfTf#g %'\\&2H2H&J\J\%]" f&8&8f>S>STzz&"<"<= 	ELL)G)GHOOPWX[\\in 	 	
 (/v7PR\']$KK))..0

4K\K\KcKcd 	 	
r2   c                 T   ||j                         }n|j                         d d }|d   }|| j                  d d d |f   }|st        | d      r-| j                  d d d |f   }|j	                  |d   |      }|}n:t        j                  |t
        j                  | j                  j                        }|| j                  |      }| j                  |      }	||	z   }
| j                  dk(  r| j                  |      }|
|z  }
| j                  |
      }
| j                  |
      }
|
S )NrA   r   r   r   r   r   )rK   r   hasattrr   r   rM   r   r   rH   r   r   r   r   r   r   )r   	input_idsr   r   inputs_embedsinput_shape
seq_lengthbuffered_token_type_ids buffered_token_type_ids_expandedr   
embeddingsr   s               r*   rq   zYosoEmbeddings.forward  s=    #..*K',,.s3K ^
,,Q^<L
 !t-.*.*=*=a*n*M'3J3Q3QR]^_R`bl3m0!A!&[

SWSdSdSkSk!l  00;M $ : :> J"%::
'':5"&":":<"H--J^^J/
\\*-
r2   )NNNNr{   r|   r}   __doc__r   rq   __classcell__r   s   @r*   r   r      s    Q
, r2   r   c                   .     e Zd Zd fd	Zd ZddZ xZS )YosoSelfAttentionc                    t         |           |j                  |j                  z  dk7  r2t	        |d      s&t        d|j                   d|j                   d      t        d u}t               rt               r|s	 t                |j                  | _        t        |j                  |j                  z        | _        | j                  | j                  z  | _        t!        j"                  |j                  | j                        | _        t!        j"                  |j                  | j                        | _        t!        j"                  |j                  | j                        | _        t!        j*                  |j,                        | _        ||n|j0                  | _        |j2                  | _        |j4                  | _        |j6                  d u| _        |j:                  | _        |j<                  | _        |j>                  | _        | j4                  | j:                  | j<                  | j>                  d| _         |j6                  Zt!        jB                  |j                  |j                  |j6                  df|j6                  d	z  dfd
|j                        | _"        y y # t        $ r#}t        j                  d|        Y d }~Fd }~ww xY w)Nr   embedding_sizezThe hidden size (z6) is not a multiple of the number of attention heads ()zGCould not load the custom kernel for multi-scale deformable attention: )rc   r   rV   r   r   r@   F)in_channelsout_channelskernel_sizepaddingbiasgroups)#r   r   r   num_attention_headsr   rL   r/   r   r   r1   	ExceptionloggerwarningrR   attention_head_sizeall_head_sizer   LinearrT   rU   rn   r   attention_probs_dropout_probr   r   use_expectationrc   conv_windowuse_convr   rV   r   
lsh_configConv2dconv)r   rj   r   kernel_loadeder   s        r*   r   zYosoSelfAttention.__init__)  so    : ::a?PVXhHi#F$6$6#7 8 445Q8  'd2"$);)=mn!# $*#=#= #&v'9'9F<V<V'V#W !558P8PPYYv1143E3EF
99V//1C1CDYYv1143E3EF
zz&"E"EF'>'J#PVPnPn 	$  &55#11**$6#11"// "//!// --	
 )		"66#77#//3++q0!411DI *=  n!hijhklmmns   =
J& &	K/KKc                     |j                         d d | j                  | j                  fz   } |j                  | }|j	                  dddd      S )NrA   r   r@   r   r   )rK   r   r   viewpermute)r   layernew_layer_shapes      r*   transpose_for_scoresz&YosoSelfAttention.transpose_for_scores\  sO    **,s+t/G/GIaIa.bb

O,}}Q1a((r2   c                 B   | j                  |      }| j                  | j                  |            }| j                  | j                  |            }| j                  |      }| j                  r| j                  ||d d d d d d f   z        }|j                         \  }	}
}}|j                  |	|
z  ||      }|j                  |	|
z  ||      }|j                  |	|
z  ||      }d|dz  z   }|j                  d      j                  |
d      j                  |	|
z  |      j                         }d}| j                  s||k  r|	|
z  |||z
  f}t        j                  |t        j                  ||j                        gd      }t        j                  |t        j                  ||j                        gd      }t        j                  |t        j                  ||j                        gd      }| j                  s| j                   rt#        ||g      \  }}| j                  r%t$        j'                  |||||| j(                        }n$t*        j'                  |||||| j(                        }| j                  s||k  r|d d d d d |f   }t#        |      }|j                  |	|
||      }| j                  r|z  }|j-                  dd	dd
      j/                         }|j                         d d | j0                  fz   } |j2                  | }|r||f}|S |f}|S )N      ?g     @r   rI       rG   rA   r   r@   r   rd   )rT   r   rU   rn   r   r   rK   rQ   	unsqueezerepeat_interleaverR   r   rM   catr   rH   trainingrE   ra   applyr   r   r   r8   r   r   )r   hidden_statesattention_maskoutput_attentionsmixed_query_layer	key_layervalue_layerquery_layerconv_value_layer
batch_size	num_headsseq_lenhead_dimgpu_warp_sizepad_sizecontext_layernew_context_layer_shapeoutputss                     r*   rq   zYosoSelfAttention.forwarda  s0    JJ}5--dhh}.EF	//

=0IJ//0AB==#yy~aqRVFV7W)WX3>3C3C3E0
Iw!))*y*@'8T%%j9&<gxP	!))*y*@'8T~77$$Q'ya0WZ)+W5SU	 	 $$(]*B!I-w8PPH))KK1C1CD K 		KK1A1AB I  ))KK1C1CD K 4==%.Y/G%H"K*00YUYUdUdM .33YUYUdUdM $$(]*B)!Q		/:M!-0%--j)WhW==--M%--aAq9DDF"/"4"4"6s";t?Q?Q>S"S***,CD4E=-0 MZK[r2   r4   NF)r{   r|   r}   r   r   rq   r   r   s   @r*   r   r   (  s    1f)
Qr2   r   c                   n     e Zd Z fdZdej
                  dej
                  dej
                  fdZ xZS )YosoSelfOutputc                 (   t         |           t        j                  |j                  |j                        | _        t        j                  |j                  |j                        | _        t        j                  |j                        | _
        y Nr   )r   r   r   r   r   denser   r   r   r   r   r   s     r*   r   zYosoSelfOutput.__init__  s`    YYv1163E3EF
f&8&8f>S>STzz&"<"<=r2   r   input_tensorreturnc                 r    | j                  |      }| j                  |      }| j                  ||z         }|S r4   r   r   r   r   r   r   s      r*   rq   zYosoSelfOutput.forward  7    

=1]3}|'CDr2   r{   r|   r}   r   rM   Tensorrq   r   r   s   @r*   r   r     1    >U\\  RWR^R^ r2   r   c                   .     e Zd Zd fd	Zd ZddZ xZS )YosoAttentionc                     t         |           t        ||      | _        t	        |      | _        t               | _        y )N)r   )r   r   r   r   r   outputsetpruned_heads)r   rj   r   r   s      r*   r   zYosoAttention.__init__  s3    %fF]^	$V,Er2   c                 >   t        |      dk(  ry t        || j                  j                  | j                  j                  | j
                        \  }}t        | j                  j                  |      | j                  _        t        | j                  j                  |      | j                  _        t        | j                  j                  |      | j                  _	        t        | j                  j                  |d      | j                  _        | j                  j                  t        |      z
  | j                  _        | j                  j                  | j                  j                  z  | j                  _        | j
                  j                  |      | _        y )Nr   r   rI   )rJ   r   r   r   r   r  r   rT   rU   rn   r  r   r   union)r   headsindexs      r*   prune_headszYosoAttention.prune_heads  s   u:?749900$))2O2OQUQbQb
u
 -TYY__eD		*499==%@		,TYY__eD		.t{{/@/@%QO )-		(E(EE
(R		%"&))"?"?$))B_B_"_		 --33E:r2   c                 h    | j                  |||      }| j                  |d   |      }|f|dd  z   }|S )Nr   r   )r   r  )r   r   r   r   self_outputsattention_outputr   s          r*   rq   zYosoAttention.forward  sC    yy@QR;;|AF#%QR(88r2   r4   r   )r{   r|   r}   r   r  rq   r   r   s   @r*   r
  r
    s    ";$r2   r
  c                   V     e Zd Z fdZdej
                  dej
                  fdZ xZS )YosoIntermediatec                    t         |           t        j                  |j                  |j
                        | _        t        |j                  t              rt        |j                     | _        y |j                  | _        y r4   )r   r   r   r   r   intermediate_sizer   r5   
hidden_actstrr   intermediate_act_fnr   s     r*   r   zYosoIntermediate.__init__  s]    YYv1163K3KL
f''-'-f.?.?'@D$'-'8'8D$r2   r   r  c                 J    | j                  |      }| j                  |      }|S r4   )r   r  r   r   s     r*   rq   zYosoIntermediate.forward  s&    

=100?r2   r  r   s   @r*   r  r    s#    9U\\ ell r2   r  c                   n     e Zd Z fdZdej
                  dej
                  dej
                  fdZ xZS )
YosoOutputc                 (   t         |           t        j                  |j                  |j
                        | _        t        j                  |j
                  |j                        | _        t        j                  |j                        | _        y r   )r   r   r   r   r  r   r   r   r   r   r   r   r   s     r*   r   zYosoOutput.__init__  s`    YYv779K9KL
f&8&8f>S>STzz&"<"<=r2   r   r   r  c                 r    | j                  |      }| j                  |      }| j                  ||z         }|S r4   r  r  s      r*   rq   zYosoOutput.forward  r  r2   r  r   s   @r*   r!  r!    r  r2   r!  c                   ,     e Zd Z fdZddZd Z xZS )	YosoLayerc                     t         |           |j                  | _        d| _        t	        |      | _        |j                  | _        t        |      | _        t        |      | _
        y Nr   )r   r   chunk_size_feed_forwardseq_len_dimr
  	attentionadd_cross_attentionr  intermediater!  r  r   s     r*   r   zYosoLayer.__init__  sW    '-'E'E$&v.#)#=#= ,V4 (r2   c                     | j                  |||      }|d   }|dd  }t        | j                  | j                  | j                  |      }|f|z   }|S )N)r   r   r   )r*  r   feed_forward_chunkr(  r)  )r   r   r   r   self_attention_outputsr  r   layer_outputs           r*   rq   zYosoLayer.forward  sh    !%~ar!s1!4(,0##T%A%A4CSCSUe
  /G+r2   c                 L    | j                  |      }| j                  ||      }|S r4   )r,  r  )r   r  intermediate_outputr0  s       r*   r.  zYosoLayer.feed_forward_chunk  s,    "//0@A{{#68HIr2   r   )r{   r|   r}   r   rq   r.  r   r   s   @r*   r%  r%    s    )r2   r%  c                   0     e Zd Z fdZ	 	 	 	 	 ddZ xZS )YosoEncoderc                     t         |           || _        t        j                  t        |j                        D cg c]  }t        |       c}      | _        d| _	        y c c}w r   )
r   r   rj   r   
ModuleListrangenum_hidden_layersr%  r   gradient_checkpointing)r   rj   _r   s      r*   r   zYosoEncoder.__init__!  sN    ]]uVE]E]?^#_!If$5#_`
&+# $`s   A#c                 b   |rdnd }|rdnd }t        | j                        D ]_  \  }	}
|r||fz   }| j                  r+| j                  r| j	                  |
j
                  |||      }n
 |
|||      }|d   }|sW||d   fz   }a |r||fz   }|st        d |||fD              S t        |||      S )Nr   r   r   c              3   &   K   | ]	  }||  y wr4   r   ).0vs     r*   	<genexpr>z&YosoEncoder.forward.<locals>.<genexpr>I  s     mq_`_lms   )last_hidden_stater   
attentions)	enumerater   r9  r   _gradient_checkpointing_func__call__tupler   )r   r   r   	head_maskr   output_hidden_statesreturn_dictall_hidden_statesall_self_attentionsilayer_modulelayer_outputss               r*   rq   zYosoEncoder.forward'  s     #7BD$5b4(4 	POA|#$58H$H!**t}} $ A A ))!"%	! !-]NL] ^)!,M &9]1=M<O&O#!	P$   1]4D Dm]4EGZ$[mmm1++*
 	
r2   )NNFFT)r{   r|   r}   r   rq   r   r   s   @r*   r4  r4     s    , "'
r2   r4  c                   V     e Zd Z fdZdej
                  dej
                  fdZ xZS )YosoPredictionHeadTransformc                 h   t         |           t        j                  |j                  |j                        | _        t        |j                  t              rt        |j                     | _
        n|j                  | _
        t        j                  |j                  |j                        | _        y r   )r   r   r   r   r   r   r5   r  r  r   transform_act_fnr   r   r   s     r*   r   z$YosoPredictionHeadTransform.__init__S  s{    YYv1163E3EF
f''-$*6+<+<$=D!$*$5$5D!f&8&8f>S>STr2   r   r  c                 l    | j                  |      }| j                  |      }| j                  |      }|S r4   )r   rQ  r   r  s     r*   rq   z#YosoPredictionHeadTransform.forward\  s4    

=1--m<}5r2   r  r   s   @r*   rO  rO  R  s$    UU\\ ell r2   rO  c                   *     e Zd Z fdZd Zd Z xZS )YosoLMPredictionHeadc                 H   t         |           t        |      | _        t	        j
                  |j                  |j                  d      | _        t	        j                  t        j                  |j                              | _        | j                  | j                  _        y )NF)r   )r   r   rO  	transformr   r   r   r   decoder	ParameterrM   r   r   r   s     r*   r   zYosoLMPredictionHead.__init__e  sm    4V< yy!3!3V5F5FUSLLV->->!?@	 !IIr2   c                 :    | j                   | j                  _         y r4   )r   rW  r   s    r*   _tie_weightsz!YosoLMPredictionHead._tie_weightsr  s     IIr2   c                 J    | j                  |      }| j                  |      }|S r4   )rV  rW  r  s     r*   rq   zYosoLMPredictionHead.forwardu  s$    }5]3r2   )r{   r|   r}   r   r[  rq   r   r   s   @r*   rT  rT  d  s    &&r2   rT  c                   V     e Zd Z fdZdej
                  dej
                  fdZ xZS )YosoOnlyMLMHeadc                 B    t         |           t        |      | _        y r4   )r   r   rT  predictionsr   s     r*   r   zYosoOnlyMLMHead.__init__}  s    /7r2   sequence_outputr  c                 (    | j                  |      }|S r4   )r`  )r   ra  prediction_scoress      r*   rq   zYosoOnlyMLMHead.forward  s     ,,_=  r2   r  r   s   @r*   r^  r^  |  s#    8!u|| ! !r2   r^  c                   "    e Zd ZdZeZdZdZd Zy)YosoPreTrainedModelz
    An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
    models.
    r#   Tc                    t        |t        j                        rm|j                  j                  j                  d| j                  j                         |j                  %|j                  j                  j                          yyt        |t        j                        rz|j                  j                  j                  d| j                  j                         |j                  2|j                  j                  |j                     j                          yyt        |t        j                        rJ|j                  j                  j                          |j                  j                  j                  d       yy)zInitialize the weightsg        )meanstdNr   )r5   r   r   weightdatanormal_rj   initializer_ranger   zero_r   r   r   fill_)r   modules     r*   _init_weightsz!YosoPreTrainedModel._init_weights  s   fbii( MM&&CT[[5R5R&S{{&  &&( '-MM&&CT[[5R5R&S!!-""6#5#56<<> .-KK""$MM$$S) .r2   N)	r{   r|   r}   r   r   config_classbase_model_prefixsupports_gradient_checkpointingrp  r   r2   r*   re  re    s    
 L&*#*r2   re  aG  
    This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use
    it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and
    behavior.

    Parameters:
        config ([`YosoConfig`]): Model configuration class with all the parameters of the model.
            Initializing with a config file does not load the weights associated with the model, only the
            configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
a5
  
    Args:
        input_ids (`torch.LongTensor` of shape `({0})`):
            Indices of input sequence tokens in the vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*):
            Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:

            - 1 for tokens that are **not masked**,
            - 0 for tokens that are **masked**.

            [What are attention masks?](../glossary#attention-mask)
        token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*):
            Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
            1]`:

            - 0 corresponds to a *sentence A* token,
            - 1 corresponds to a *sentence B* token.

            [What are token type IDs?](../glossary#token-type-ids)
        position_ids (`torch.LongTensor` of shape `({0})`, *optional*):
            Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
            config.max_position_embeddings - 1]`.

            [What are position IDs?](../glossary#position-ids)
        head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*):
            Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`:

            - 1 indicates the head is **not masked**,
            - 0 indicates the head is **masked**.

        inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*):
            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
            is useful if you want more control over how to convert *input_ids* indices into associated vectors than the
            model's internal embedding lookup matrix.
        output_attentions (`bool`, *optional*):
            Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
            tensors for more detail.
        output_hidden_states (`bool`, *optional*):
            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
            more detail.
        return_dict (`bool`, *optional*):
            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
z^The bare YOSO Model transformer outputting raw hidden-states without any specific head on top.c                   |    e Zd Z fdZd Zd Zd Z eej                  d             e
eee      	 	 	 	 	 	 	 	 	 ddeej                      deej                      d	eej                      d
eej                      deej                      deej                      dee   dee   dee   deeef   fd              Z xZS )	YosoModelc                     t         |   |       || _        t        |      | _        t        |      | _        | j                          y r4   )r   r   rj   r   r   r4  encoder	post_initr   s     r*   r   zYosoModel.__init__  s;     (0"6* 	r2   c                 .    | j                   j                  S r4   r   r   rZ  s    r*   get_input_embeddingszYosoModel.get_input_embeddings  s    ...r2   c                 &    || j                   _        y r4   rz  )r   rn   s     r*   set_input_embeddingszYosoModel.set_input_embeddings  s    */'r2   c                     |j                         D ]7  \  }}| j                  j                  |   j                  j	                  |       9 y)z
        Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base
        class PreTrainedModel
        N)itemsrw  r   r*  r  )r   heads_to_pruner   r  s       r*   _prune_headszYosoModel._prune_heads  sE    
 +002 	CLE5LLu%//;;EB	Cr2   batch_size, sequence_length
checkpointoutput_typerq  r   r   r   r   rF  r   r   rG  rH  r  c
                    ||n| j                   j                  }||n| j                   j                  }|	|	n| j                   j                  }	||t	        d      |#| j                  ||       |j                         }
n!||j                         d d }
nt	        d      |
\  }}||j                  n|j                  }|t        j                  ||f|      }|pt        | j                  d      r4| j                  j                  d d d |f   }|j                  ||      }|}n&t        j                  |
t        j                  |      }| j!                  || j                   j"                        }| j                  ||||      }| j%                  ||||||	      }|d	   }|	s	|f|d
d  z   S t'        ||j(                  |j*                  |j,                        S )NzDYou cannot specify both input_ids and inputs_embeds at the same timerA   z5You have to specify either input_ids or inputs_embedsrG   r   r   )r   r   r   r   )r   rF  r   rG  rH  r   r   )r@  r   rA  cross_attentions)rj   r   rG  use_return_dictrL   %warn_if_padding_and_no_attention_maskrK   rH   rM   onesr   r   r   r   r   r   get_head_maskr8  rw  r   r   rA  r  )r   r   r   r   r   rF  r   r   rG  rH  r   r   r   rH   r   r   embedding_outputencoder_outputsra  s                      r*   rq   zYosoModel.forward  s   $ 2C1N-TXT_T_TqTq$8$D $++JjJj 	 &1%<k$++B]B] ]%>cdd"66y.Q#..*K&',,.s3KTUU!,
J%.%:!!@T@T!"ZZ*j)A6RN!t(89*.//*H*HKZK*X'3J3Q3QR\^h3i0!A!&[

SY!Z &&y$++2O2OP	??%)'	 + 
 ,,)/!5# ' 
 *!,#%(;;;1-)77&11,==	
 	
r2   )	NNNNNNNNN)r{   r|   r}   r   r{  r}  r  r   YOSO_INPUTS_DOCSTRINGformatr   _CHECKPOINT_FOR_DOCr   _CONFIG_FOR_DOCr   rM   r  boolr   r   rq   r   r   s   @r*   ru  ru    s-   
/0C ++@+G+GHe+fg&6$ -11515/3,004,0/3&*I
ELL)I
 !.I
 !.	I

 u||,I
 ELL)I
  -I
 $D>I
 'tnI
 d^I
 
u88	9I
 hI
r2   ru  z2YOSO Model with a `language modeling` head on top.c                       e Zd ZddgZ fdZd Zd Z eej                  d             e
eee      	 	 	 	 	 	 	 	 	 	 ddeej                      d	eej                      d
eej                      deej                      deej                      deej                      deej                      dee   dee   dee   deeef   fd              Z xZS )YosoForMaskedLMzcls.predictions.decoder.weightzcls.predictions.decoder.biasc                     t         |   |       t        |      | _        t	        |      | _        | j                          y r4   )r   r   ru  r#   r^  clsrx  r   s     r*   r   zYosoForMaskedLM.__init__Q  s4     f%	"6* 	r2   c                 B    | j                   j                  j                  S r4   )r  r`  rW  rZ  s    r*   get_output_embeddingsz%YosoForMaskedLM.get_output_embeddingsZ  s    xx##+++r2   c                     || j                   j                  _        |j                  | j                   j                  _        y r4   )r  r`  rW  r   )r   new_embeddingss     r*   set_output_embeddingsz%YosoForMaskedLM.set_output_embeddings]  s,    '5$$2$7$7!r2   r  r  r   r   r   r   rF  r   labelsr   rG  rH  r  c                    |
|
n| j                   j                  }
| j                  ||||||||	|
	      }|d   }| j                  |      }d}|Ft	               } ||j                  d| j                   j                        |j                  d            }|
s|f|dd z   }||f|z   S |S t        |||j                  |j                        S )a  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
            config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
            loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
        Nr   r   r   rF  r   r   rG  rH  r   rA   r   losslogitsr   rA  )
rj   r  r#   r  r	   r   r   r   r   rA  )r   r   r   r   r   rF  r   r  r   rG  rH  r   ra  rc  masked_lm_lossloss_fctr  s                    r*   rq   zYosoForMaskedLM.forwarda  s    2 &1%<k$++B]B]))))%'/!5#  

 "!* HH_5')H%&7&<&<RAWAW&XZ`ZeZefhZijN')GABK7F3A3M^%.YSYY$!//))	
 	
r2   
NNNNNNNNNN)r{   r|   r}   _tied_weights_keysr   r  r  r   r  r  r   r  r   r  r   rM   r  r  r   r   rq   r   r   s   @r*   r  r  M  s<   :<Z[,8 ++@+G+GHe+fg&"$ -11515/3,004)-,0/3&*2
ELL)2
 !.2
 !.	2

 u||,2
 ELL)2
  -2
 &2
 $D>2
 'tn2
 d^2
 
un$	%2
 h2
r2   r  c                   (     e Zd ZdZ fdZd Z xZS )YosoClassificationHeadz-Head for sentence-level classification tasks.c                 4   t         |           t        j                  |j                  |j                        | _        t        j                  |j                        | _        t        j                  |j                  |j                        | _
        || _        y r4   )r   r   r   r   r   r   r   r   r   
num_labelsout_projrj   r   s     r*   r   zYosoClassificationHead.__init__  sg    YYv1163E3EF
zz&"<"<=		&"4"4f6G6GHr2   c                     |d d dd d f   }| j                  |      }| j                  |      }t        | j                  j                     |      }| j                  |      }| j                  |      }|S )Nr   )r   r   r   rj   r  r  )r   featureskwargsxs       r*   rq   zYosoClassificationHead.forward  se    Q1WLLOJJqM4;;))*1-LLOMM!r2   r   r   s   @r*   r  r    s    7r2   r  zYOSO Model transformer with a sequence classification/regression head on top (a linear layer on top of
    the pooled output) e.g. for GLUE tasks.c                       e Zd Z fdZ eej                  d             eee	e
      	 	 	 	 	 	 	 	 	 	 ddeej                     deej                     deej                     deej                     deej                     d	eej                     d
eej                     dee   dee   dee   deee	f   fd              Z xZS )YosoForSequenceClassificationc                     t         |   |       |j                  | _        t        |      | _        t        |      | _        | j                          y r4   )r   r   r  ru  r#   r  
classifierrx  r   s     r*   r   z&YosoForSequenceClassification.__init__  sA      ++f%	08 	r2   r  r  r   r   r   r   rF  r   r  r   rG  rH  r  c                    |
|
n| j                   j                  }
| j                  ||||||||	|
	      }|d   }| j                  |      }d}|| j                   j                  | j
                  dk(  rd| j                   _        nl| j
                  dkD  rL|j                  t        j                  k(  s|j                  t        j                  k(  rd| j                   _        nd| j                   _        | j                   j                  dk(  rIt               }| j
                  dk(  r& ||j                         |j                               }n |||      }n| j                   j                  dk(  r=t               } ||j                  d| j
                        |j                  d            }n,| j                   j                  dk(  rt               } |||      }|
s|f|dd z   }||f|z   S |S t        |||j                   |j"                  	      S )
a  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        Nr  r   r   
regressionsingle_label_classificationmulti_label_classificationrA   r  )rj   r  r#   r  problem_typer  r   rM   r   rR   r
   squeezer	   r   r   r   r   rA  )r   r   r   r   r   rF  r   r  r   rG  rH  r   ra  r  r  r  r  s                    r*   rq   z%YosoForSequenceClassification.forward  s   2 &1%<k$++B]B]))))%'/!5#  

 "!*1{{''/??a'/;DKK,__q(fllejj.HFLL\a\e\eLe/LDKK,/KDKK,{{''<7"9??a'#FNN$4fnn6FGD#FF3D))-JJ+-B @&++b/R))-II,./Y,F)-)9TGf$EvE'!//))	
 	
r2   r  )r{   r|   r}   r   r   r  r  r   r  r   r  r   rM   r  r  r   r   rq   r   r   s   @r*   r  r    s4    ++@+G+GHe+fg&,$ -11515/3,004)-,0/3&*C
ELL)C
 !.C
 !.	C

 u||,C
 ELL)C
  -C
 &C
 $D>C
 'tnC
 d^C
 
u..	/C
 hC
r2   r  zYOSO Model with a multiple choice classification head on top (a linear layer on top of
    the pooled output and a softmax) e.g. for RocStories/SWAG tasks.c                       e Zd Z fdZ eej                  d             eee	e
      	 	 	 	 	 	 	 	 	 	 ddeej                     deej                     deej                     deej                     deej                     d	eej                     d
eej                     dee   dee   dee   deee	f   fd              Z xZS )YosoForMultipleChoicec                    t         |   |       t        |      | _        t	        j
                  |j                  |j                        | _        t	        j
                  |j                  d      | _        | j                          y r'  )
r   r   ru  r#   r   r   r   pre_classifierr  rx  r   s     r*   r   zYosoForMultipleChoice.__init__  s_     f%	 ii(:(:F<N<NO))F$6$6: 	r2   z(batch_size, num_choices, sequence_lengthr  r   r   r   r   rF  r   r  r   rG  rH  r  c                    |
|
n| j                   j                  }
||j                  d   n|j                  d   }|!|j                  d|j	                  d            nd}|!|j                  d|j	                  d            nd}|!|j                  d|j	                  d            nd}|!|j                  d|j	                  d            nd}|1|j                  d|j	                  d      |j	                  d            nd}| j                  ||||||||	|
	      }|d   }|dddf   }| j                  |      } t        j                         |      }| j                  |      }|j                  d|      }d}|t               } |||      }|
s|f|dd z   }||f|z   S |S t        |||j                  |j                        S )aJ  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the multiple choice classification loss. Indices should be in `[0, ...,
            num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See
            `input_ids` above)
        Nr   rA   rd   r  r   r  )rj   r  shaper   rK   r#   r  r   ReLUr  r	   r   r   rA  )r   r   r   r   r   rF  r   r  r   rG  rH  num_choicesr   hidden_statepooled_outputr  reshaped_logitsr  r  r  s                       r*   rq   zYosoForMultipleChoice.forward  s   2 &1%<k$++B]B],5,Aiooa(}GZGZ[\G]>G>SINN2y~~b'9:Y]	M[Mg,,R1D1DR1HImqM[Mg,,R1D1DR1HImqGSG_|((\->->r-BCei ( r=#5#5b#9=;M;Mb;QR 	 ))))%'/!5#  

 qz$QT*++M:!	-0/ ++b+6')HOV4D%''!"+5F)-)9TGf$EvE("!//))	
 	
r2   r  )r{   r|   r}   r   r   r  r  r   r  r   r  r   rM   r  r  r   r   rq   r   r   s   @r*   r  r    s4    ++@+G+GHr+st&-$ -11515/3,004)-,0/3&*B
ELL)B
 !.B
 !.	B

 u||,B
 ELL)B
  -B
 &B
 $D>B
 'tnB
 d^B
 
u//	0B
 uB
r2   r  zYOSO Model with a token classification head on top (a linear layer on top of
    the hidden-states output) e.g. for Named-Entity-Recognition (NER) tasks.c                       e Zd Z fdZ eej                  d             eee	e
      	 	 	 	 	 	 	 	 	 	 ddeej                     deej                     deej                     deej                     deej                     d	eej                     d
eej                     dee   dee   dee   deee	f   fd              Z xZS )YosoForTokenClassificationc                 ,   t         |   |       |j                  | _        t        |      | _        t        j                  |j                        | _        t        j                  |j                  |j                        | _        | j                          y r4   )r   r   r  ru  r#   r   r   r   r   r   r   r  rx  r   s     r*   r   z#YosoForTokenClassification.__init__m  si      ++f%	zz&"<"<=))F$6$68I8IJ 	r2   r  r  r   r   r   r   rF  r   r  r   rG  rH  r  c                    |
|
n| j                   j                  }
| j                  ||||||||	|
	      }|d   }| j                  |      }| j	                  |      }d}|t               }||j                  d      dk(  }|j                  d| j                        }t        j                  ||j                  d      t        j                  |j                        j                  |            } |||      }n2 ||j                  d| j                        |j                  d            }|
s|f|dd z   }||f|z   S |S t        |||j                  |j                        S )z
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`.
        Nr  r   rA   r   r  )rj   r  r#   r   r  r	   r   r  rM   wherer=   ignore_indextype_asr   r   rA  )r   r   r   r   r   rF  r   r  r   rG  rH  r   ra  r  r  r  active_lossactive_logitsactive_labelsr  s                       r*   rq   z"YosoForTokenClassification.forwardx  sk   . &1%<k$++B]B]))))%'/!5#  

 "!*,,71')H),11"5: &B @ %R%,,x?T?T2U2]2]^d2e!  }=B @&++b/RY,F)-)9TGf$EvE$!//))	
 	
r2   r  )r{   r|   r}   r   r   r  r  r   r  r   r  r   rM   r  r  r   r   rq   r   r   s   @r*   r  r  g  s'   	 ++@+G+GHe+fg&)$ -11515/3,004)-,0/3&*;
ELL);
 !.;
 !.	;

 u||,;
 ELL);
  -;
 &;
 $D>;
 'tn;
 d^;
 
u++	,;
 h;
r2   r  zYOSO Model with a span classification head on top for extractive question-answering tasks like SQuAD (a linear
    layers on top of the hidden-states output to compute `span start logits` and `span end logits`).c                       e Zd Z fdZ eej                  d             eee	e
      	 	 	 	 	 	 	 	 	 	 	 ddeej                     deej                     deej                     deej                     deej                     d	eej                     d
eej                     deej                     dee   dee   dee   deee	f   fd              Z xZS )YosoForQuestionAnsweringc                     t         |   |       d|_        |j                  | _        t        |      | _        t        j                  |j                  |j                        | _        | j                          y )Nr@   )
r   r   r  ru  r#   r   r   r   
qa_outputsrx  r   s     r*   r   z!YosoForQuestionAnswering.__init__  s[      ++f%	))F$6$68I8IJ 	r2   r  r  r   r   r   r   rF  r   start_positionsend_positionsr   rG  rH  r  c                    ||n| j                   j                  }| j                  |||||||	|
|	      }|d   }| j                  |      }|j	                  dd      \  }}|j                  d      }|j                  d      }d}||t        |j                               dkD  r|j                  d      }t        |j                               dkD  r|j                  d      }|j                  d      }|j                  d|      }|j                  d|      }t        |      } |||      } |||      }||z   dz  }|s||f|dd z   }||f|z   S |S t        ||||j                  |j                  	      S )
a  
        start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for position (index) of the start of the labelled span for computing the token classification loss.
            Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
            are not taken into account for computing the loss.
        end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for position (index) of the end of the labelled span for computing the token classification loss.
            Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
            are not taken into account for computing the loss.
        Nr  r   r   rA   rI   )r  r@   )r  start_logits
end_logitsr   rA  )rj   r  r#   r  splitr  rJ   rK   clampr	   r   r   rA  )r   r   r   r   r   rF  r   r  r  r   rG  rH  r   ra  r  r  r  
total_lossignored_indexr  
start_lossend_lossr  s                          r*   rq   z YosoForQuestionAnswering.forward  s   < &1%<k$++B]B]))))%'/!5#  

 "!*1#)<<r<#: j#++B/''+

&=+D?'')*Q."1"9"9""==%%'(1, - 5 5b 9(--a0M-33A}EO)//=AM']CH!,@J
M:H$x/14J"J/'!"+=F/9/EZMF*Q6Q+%!!//))
 	
r2   )NNNNNNNNNNN)r{   r|   r}   r   r   r  r  r   r  r   r  r   rM   r  r  r   r   rq   r   r   s   @r*   r  r    sK   
 ++@+G+GHe+fg&0$ -11515/3,0042604,0/3&*H
ELL)H
 !.H
 !.	H

 u||,H
 ELL)H
  -H
 "%,,/H
  -H
 $D>H
 'tnH
 d^H
 
u22	3H
 hH
r2   r  )Kr   rg   pathlibr   typingr   r   r   rM   torch.utils.checkpointr   torch.nnr   r	   r
   activationsr   modeling_outputsr   r   r   r   r   r   modeling_utilsr   pytorch_utilsr   r   r   utilsr   r   r   r   r   r   configuration_yosor   
get_loggerr{   r   r  r  r/   r1   r>   rE   r_   autogradFunctionra   r   Moduler   r   r   r
  r  r!  r%  r4  rO  rT  r^  re  YOSO_START_DOCSTRINGr  ru  r  r  r  r  r  r  r   r2   r*   <module>r     s      ) )    A A !  . l l  + 
		H	%,  1C.&BU^^,, B>VB// VBt9RYY 9xJ		 J\RYY BII Bryy   		 :.
")) .
d")) $299 0!bii !*/ *6	 / d dh
# h
	h
V NPdeK
) K
 fK
\RYY * /
S
$7 S

S
l H
S
/ S

S
l P
M
!4 M

M
` h
[
2 [

[
r2   