
    sg,                     ,   d dl mZmZ  G d de      Z G d de      Z G d de      Z G d	 d
e      Z G d de      Z G d de      Z G d de      Z	 G d de      Z
 G d de      Z G d de      Z G d de      Z G d de      Z G d de      Z G d de      Z G d d e      Z G d! d"e      Z G d# d$e      Z G d% d&e      Z G d' d(e      Z G d) d*e      Z G d+ d,e      Z G d- d.e      Z G d/ d0e      Z G d1 d2e      Z G d3 d4e      Z G d5 d6e      Z G d7 d8e      Z G d9 d:e      Z G d; d<e      Z G d= d>e      Z  G d? d@e      Z! G dA dBe      Z" G dC dDe      Z# G dE dFe      Z$ G dG dHe      Z% G dI dJe      Z& G dK dLe      Z' G dM dNe      Z( G dO dPe      Z) G dQ dRe      Z* G dS dTe      Z+ G dU dVe      Z, G dW dXe      Z- G dY dZe      Z. G d[ d\e      Z/ G d] d^e      Z0 G d_ d`e      Z1 G da dbe      Z2 G dc dde      Z3 G de dfe      Z4 G dg dhe      Z5 G di dje      Z6 G dk dle      Z7 G dm dne      Z8 G do dpe      Z9 G dq dre      Z: G ds dte      Z; G du dve      Z< G dw dxe      Z= G dy dze      Z> G d{ d|e      Z? G d} d~e      Z@ G d de      ZA G d de      ZB G d de      ZCy)   )DummyObjectrequires_backendsc                       e Zd ZdgZd Zy)AlbertTokenizerFast
tokenizersc                     t        | dg       y Nr   r   selfargskwargss      ^/var/www/html/venv/lib/python3.12/site-packages/transformers/utils/dummy_tokenizers_objects.py__init__zAlbertTokenizerFast.__init__       $/    N__name__
__module____qualname__	_backendsr    r   r   r   r          I0r   r   )	metaclassc                       e Zd ZdgZd Zy)BartTokenizerFastr   c                     t        | dg       y r	   r
   r   s      r   r   zBartTokenizerFast.__init__   r   r   Nr   r   r   r   r   r      r   r   r   c                       e Zd ZdgZd Zy)BarthezTokenizerFastr   c                     t        | dg       y r	   r
   r   s      r   r   zBarthezTokenizerFast.__init__   r   r   Nr   r   r   r   r   r      r   r   r   c                       e Zd ZdgZd Zy)BertTokenizerFastr   c                     t        | dg       y r	   r
   r   s      r   r   zBertTokenizerFast.__init__   r   r   Nr   r   r   r   r"   r"      r   r   r"   c                       e Zd ZdgZd Zy)BigBirdTokenizerFastr   c                     t        | dg       y r	   r
   r   s      r   r   zBigBirdTokenizerFast.__init__$   r   r   Nr   r   r   r   r%   r%   !   r   r   r%   c                       e Zd ZdgZd Zy)BlenderbotTokenizerFastr   c                     t        | dg       y r	   r
   r   s      r   r   z BlenderbotTokenizerFast.__init__+   r   r   Nr   r   r   r   r(   r(   (   r   r   r(   c                       e Zd ZdgZd Zy)BlenderbotSmallTokenizerFastr   c                     t        | dg       y r	   r
   r   s      r   r   z%BlenderbotSmallTokenizerFast.__init__2   r   r   Nr   r   r   r   r+   r+   /   r   r   r+   c                       e Zd ZdgZd Zy)BloomTokenizerFastr   c                     t        | dg       y r	   r
   r   s      r   r   zBloomTokenizerFast.__init__9   r   r   Nr   r   r   r   r.   r.   6   r   r   r.   c                       e Zd ZdgZd Zy)CamembertTokenizerFastr   c                     t        | dg       y r	   r
   r   s      r   r   zCamembertTokenizerFast.__init__@   r   r   Nr   r   r   r   r1   r1   =   r   r   r1   c                       e Zd ZdgZd Zy)CLIPTokenizerFastr   c                     t        | dg       y r	   r
   r   s      r   r   zCLIPTokenizerFast.__init__G   r   r   Nr   r   r   r   r4   r4   D   r   r   r4   c                       e Zd ZdgZd Zy)CodeLlamaTokenizerFastr   c                     t        | dg       y r	   r
   r   s      r   r   zCodeLlamaTokenizerFast.__init__N   r   r   Nr   r   r   r   r7   r7   K   r   r   r7   c                       e Zd ZdgZd Zy)CodeGenTokenizerFastr   c                     t        | dg       y r	   r
   r   s      r   r   zCodeGenTokenizerFast.__init__U   r   r   Nr   r   r   r   r:   r:   R   r   r   r:   c                       e Zd ZdgZd Zy)CohereTokenizerFastr   c                     t        | dg       y r	   r
   r   s      r   r   zCohereTokenizerFast.__init__\   r   r   Nr   r   r   r   r=   r=   Y   r   r   r=   c                       e Zd ZdgZd Zy)ConvBertTokenizerFastr   c                     t        | dg       y r	   r
   r   s      r   r   zConvBertTokenizerFast.__init__c   r   r   Nr   r   r   r   r@   r@   `   r   r   r@   c                       e Zd ZdgZd Zy)CpmTokenizerFastr   c                     t        | dg       y r	   r
   r   s      r   r   zCpmTokenizerFast.__init__j   r   r   Nr   r   r   r   rC   rC   g   r   r   rC   c                       e Zd ZdgZd Zy)DebertaTokenizerFastr   c                     t        | dg       y r	   r
   r   s      r   r   zDebertaTokenizerFast.__init__q   r   r   Nr   r   r   r   rF   rF   n   r   r   rF   c                       e Zd ZdgZd Zy)DebertaV2TokenizerFastr   c                     t        | dg       y r	   r
   r   s      r   r   zDebertaV2TokenizerFast.__init__x   r   r   Nr   r   r   r   rI   rI   u   r   r   rI   c                       e Zd ZdgZd Zy)RealmTokenizerFastr   c                     t        | dg       y r	   r
   r   s      r   r   zRealmTokenizerFast.__init__   r   r   Nr   r   r   r   rL   rL   |   r   r   rL   c                       e Zd ZdgZd Zy)RetriBertTokenizerFastr   c                     t        | dg       y r	   r
   r   s      r   r   zRetriBertTokenizerFast.__init__   r   r   Nr   r   r   r   rO   rO      r   r   rO   c                       e Zd ZdgZd Zy)DistilBertTokenizerFastr   c                     t        | dg       y r	   r
   r   s      r   r   z DistilBertTokenizerFast.__init__   r   r   Nr   r   r   r   rR   rR      r   r   rR   c                       e Zd ZdgZd Zy)DPRContextEncoderTokenizerFastr   c                     t        | dg       y r	   r
   r   s      r   r   z'DPRContextEncoderTokenizerFast.__init__   r   r   Nr   r   r   r   rU   rU      r   r   rU   c                       e Zd ZdgZd Zy)DPRQuestionEncoderTokenizerFastr   c                     t        | dg       y r	   r
   r   s      r   r   z(DPRQuestionEncoderTokenizerFast.__init__   r   r   Nr   r   r   r   rX   rX      r   r   rX   c                       e Zd ZdgZd Zy)DPRReaderTokenizerFastr   c                     t        | dg       y r	   r
   r   s      r   r   zDPRReaderTokenizerFast.__init__   r   r   Nr   r   r   r   r[   r[      r   r   r[   c                       e Zd ZdgZd Zy)ElectraTokenizerFastr   c                     t        | dg       y r	   r
   r   s      r   r   zElectraTokenizerFast.__init__   r   r   Nr   r   r   r   r^   r^      r   r   r^   c                       e Zd ZdgZd Zy)FNetTokenizerFastr   c                     t        | dg       y r	   r
   r   s      r   r   zFNetTokenizerFast.__init__   r   r   Nr   r   r   r   ra   ra      r   r   ra   c                       e Zd ZdgZd Zy)FunnelTokenizerFastr   c                     t        | dg       y r	   r
   r   s      r   r   zFunnelTokenizerFast.__init__   r   r   Nr   r   r   r   rd   rd      r   r   rd   c                       e Zd ZdgZd Zy)GemmaTokenizerFastr   c                     t        | dg       y r	   r
   r   s      r   r   zGemmaTokenizerFast.__init__   r   r   Nr   r   r   r   rg   rg      r   r   rg   c                       e Zd ZdgZd Zy)GPT2TokenizerFastr   c                     t        | dg       y r	   r
   r   s      r   r   zGPT2TokenizerFast.__init__   r   r   Nr   r   r   r   rj   rj      r   r   rj   c                       e Zd ZdgZd Zy)GPTNeoXTokenizerFastr   c                     t        | dg       y r	   r
   r   s      r   r   zGPTNeoXTokenizerFast.__init__   r   r   Nr   r   r   r   rm   rm      r   r   rm   c                       e Zd ZdgZd Zy)GPTNeoXJapaneseTokenizerr   c                     t        | dg       y r	   r
   r   s      r   r   z!GPTNeoXJapaneseTokenizer.__init__   r   r   Nr   r   r   r   rp   rp      r   r   rp   c                       e Zd ZdgZd Zy)HerbertTokenizerFastr   c                     t        | dg       y r	   r
   r   s      r   r   zHerbertTokenizerFast.__init__   r   r   Nr   r   r   r   rs   rs      r   r   rs   c                       e Zd ZdgZd Zy)LayoutLMTokenizerFastr   c                     t        | dg       y r	   r
   r   s      r   r   zLayoutLMTokenizerFast.__init__   r   r   Nr   r   r   r   rv   rv      r   r   rv   c                       e Zd ZdgZd Zy)LayoutLMv2TokenizerFastr   c                     t        | dg       y r	   r
   r   s      r   r   z LayoutLMv2TokenizerFast.__init__   r   r   Nr   r   r   r   ry   ry      r   r   ry   c                       e Zd ZdgZd Zy)LayoutLMv3TokenizerFastr   c                     t        | dg       y r	   r
   r   s      r   r   z LayoutLMv3TokenizerFast.__init__   r   r   Nr   r   r   r   r|   r|      r   r   r|   c                       e Zd ZdgZd Zy)LayoutXLMTokenizerFastr   c                     t        | dg       y r	   r
   r   s      r   r   zLayoutXLMTokenizerFast.__init__   r   r   Nr   r   r   r   r   r      r   r   r   c                       e Zd ZdgZd Zy)LEDTokenizerFastr   c                     t        | dg       y r	   r
   r   s      r   r   zLEDTokenizerFast.__init__   r   r   Nr   r   r   r   r   r      r   r   r   c                       e Zd ZdgZd Zy)LlamaTokenizerFastr   c                     t        | dg       y r	   r
   r   s      r   r   zLlamaTokenizerFast.__init__  r   r   Nr   r   r   r   r   r     r   r   r   c                       e Zd ZdgZd Zy)LongformerTokenizerFastr   c                     t        | dg       y r	   r
   r   s      r   r   z LongformerTokenizerFast.__init__  r   r   Nr   r   r   r   r   r     r   r   r   c                       e Zd ZdgZd Zy)LxmertTokenizerFastr   c                     t        | dg       y r	   r
   r   s      r   r   zLxmertTokenizerFast.__init__  r   r   Nr   r   r   r   r   r     r   r   r   c                       e Zd ZdgZd Zy)MarkupLMTokenizerFastr   c                     t        | dg       y r	   r
   r   s      r   r   zMarkupLMTokenizerFast.__init__  r   r   Nr   r   r   r   r   r     r   r   r   c                       e Zd ZdgZd Zy)MBartTokenizerFastr   c                     t        | dg       y r	   r
   r   s      r   r   zMBartTokenizerFast.__init__   r   r   Nr   r   r   r   r   r     r   r   r   c                       e Zd ZdgZd Zy)MBart50TokenizerFastr   c                     t        | dg       y r	   r
   r   s      r   r   zMBart50TokenizerFast.__init__'  r   r   Nr   r   r   r   r   r   $  r   r   r   c                       e Zd ZdgZd Zy)MobileBertTokenizerFastr   c                     t        | dg       y r	   r
   r   s      r   r   z MobileBertTokenizerFast.__init__.  r   r   Nr   r   r   r   r   r   +  r   r   r   c                       e Zd ZdgZd Zy)MPNetTokenizerFastr   c                     t        | dg       y r	   r
   r   s      r   r   zMPNetTokenizerFast.__init__5  r   r   Nr   r   r   r   r   r   2  r   r   r   c                       e Zd ZdgZd Zy)MT5TokenizerFastr   c                     t        | dg       y r	   r
   r   s      r   r   zMT5TokenizerFast.__init__<  r   r   Nr   r   r   r   r   r   9  r   r   r   c                       e Zd ZdgZd Zy)MvpTokenizerFastr   c                     t        | dg       y r	   r
   r   s      r   r   zMvpTokenizerFast.__init__C  r   r   Nr   r   r   r   r   r   @  r   r   r   c                       e Zd ZdgZd Zy)NllbTokenizerFastr   c                     t        | dg       y r	   r
   r   s      r   r   zNllbTokenizerFast.__init__J  r   r   Nr   r   r   r   r   r   G  r   r   r   c                       e Zd ZdgZd Zy)NougatTokenizerFastr   c                     t        | dg       y r	   r
   r   s      r   r   zNougatTokenizerFast.__init__Q  r   r   Nr   r   r   r   r   r   N  r   r   r   c                       e Zd ZdgZd Zy)OpenAIGPTTokenizerFastr   c                     t        | dg       y r	   r
   r   s      r   r   zOpenAIGPTTokenizerFast.__init__X  r   r   Nr   r   r   r   r   r   U  r   r   r   c                       e Zd ZdgZd Zy)PegasusTokenizerFastr   c                     t        | dg       y r	   r
   r   s      r   r   zPegasusTokenizerFast.__init___  r   r   Nr   r   r   r   r   r   \  r   r   r   c                       e Zd ZdgZd Zy)Qwen2TokenizerFastr   c                     t        | dg       y r	   r
   r   s      r   r   zQwen2TokenizerFast.__init__f  r   r   Nr   r   r   r   r   r   c  r   r   r   c                       e Zd ZdgZd Zy)ReformerTokenizerFastr   c                     t        | dg       y r	   r
   r   s      r   r   zReformerTokenizerFast.__init__m  r   r   Nr   r   r   r   r   r   j  r   r   r   c                       e Zd ZdgZd Zy)RemBertTokenizerFastr   c                     t        | dg       y r	   r
   r   s      r   r   zRemBertTokenizerFast.__init__t  r   r   Nr   r   r   r   r   r   q  r   r   r   c                       e Zd ZdgZd Zy)RobertaTokenizerFastr   c                     t        | dg       y r	   r
   r   s      r   r   zRobertaTokenizerFast.__init__{  r   r   Nr   r   r   r   r   r   x  r   r   r   c                       e Zd ZdgZd Zy)RoFormerTokenizerFastr   c                     t        | dg       y r	   r
   r   s      r   r   zRoFormerTokenizerFast.__init__  r   r   Nr   r   r   r   r   r     r   r   r   c                       e Zd ZdgZd Zy)SeamlessM4TTokenizerFastr   c                     t        | dg       y r	   r
   r   s      r   r   z!SeamlessM4TTokenizerFast.__init__  r   r   Nr   r   r   r   r   r     r   r   r   c                       e Zd ZdgZd Zy)SplinterTokenizerFastr   c                     t        | dg       y r	   r
   r   s      r   r   zSplinterTokenizerFast.__init__  r   r   Nr   r   r   r   r   r     r   r   r   c                       e Zd ZdgZd Zy)SqueezeBertTokenizerFastr   c                     t        | dg       y r	   r
   r   s      r   r   z!SqueezeBertTokenizerFast.__init__  r   r   Nr   r   r   r   r   r     r   r   r   c                       e Zd ZdgZd Zy)T5TokenizerFastr   c                     t        | dg       y r	   r
   r   s      r   r   zT5TokenizerFast.__init__  r   r   Nr   r   r   r   r   r     r   r   r   c                       e Zd ZdgZd Zy)UdopTokenizerFastr   c                     t        | dg       y r	   r
   r   s      r   r   zUdopTokenizerFast.__init__  r   r   Nr   r   r   r   r   r     r   r   r   c                       e Zd ZdgZd Zy)WhisperTokenizerFastr   c                     t        | dg       y r	   r
   r   s      r   r   zWhisperTokenizerFast.__init__  r   r   Nr   r   r   r   r   r     r   r   r   c                       e Zd ZdgZd Zy)XGLMTokenizerFastr   c                     t        | dg       y r	   r
   r   s      r   r   zXGLMTokenizerFast.__init__  r   r   Nr   r   r   r   r   r     r   r   r   c                       e Zd ZdgZd Zy)XLMRobertaTokenizerFastr   c                     t        | dg       y r	   r
   r   s      r   r   z XLMRobertaTokenizerFast.__init__  r   r   Nr   r   r   r   r   r     r   r   r   c                       e Zd ZdgZd Zy)XLNetTokenizerFastr   c                     t        | dg       y r	   r
   r   s      r   r   zXLNetTokenizerFast.__init__  r   r   Nr   r   r   r   r   r     r   r   r   c                       e Zd ZdgZd Zy)PreTrainedTokenizerFastr   c                     t        | dg       y r	   r
   r   s      r   r   z PreTrainedTokenizerFast.__init__  r   r   Nr   r   r   r   r   r     r   r   r   N)Dutilsr   r   r   r   r   r"   r%   r(   r+   r.   r1   r4   r7   r:   r=   r@   rC   rF   rI   rL   rO   rR   rU   rX   r[   r^   ra   rd   rg   rj   rm   rp   rs   rv   ry   r|   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   <module>r      s   20K 00+ 00[ 00+ 00[ 00 00[ 00; 00{ 00+ 00{ 00[ 00K 00k 00 00[ 00{ 00; 00{ 00 00{ 00 00{ 00[ 00+ 00K 00; 00+ 00[ 00 00[ 00k 00 00 00{ 00 00; 00 00K 00k 00; 00[ 00 00; 00 00 00+ 00K 00{ 00[ 00; 00k 00[ 00[ 00k 00 00k 00 00 00+ 00[ 00+ 00 00; 00 0r   