
    sgD>                         d dl Z d dlmZ d dlmZmZmZ d dlmZ ddl	m
Z
mZ ddlmZ ddlmZmZ  e       rd	d
lmZ ndZ ej&                  e      ZdddZg dZ G d de      Zy)    N)copyfile)ListOptionalTuple)
processors   )
AddedTokenBatchEncoding)PreTrainedTokenizerFast)is_sentencepiece_availablelogging   )NllbTokenizerzsentencepiece.bpe.modelztokenizer.json)
vocab_filetokenizer_file)ace_Arabace_Latnacm_Arabacq_Arabaeb_Arabafr_Latnajp_Arabaka_Latnamh_Ethiapc_Arabarb_Arabars_Arabary_Arabarz_Arabasm_Bengast_Latnawa_Devaayr_Latnazb_Arabazj_Latnbak_Cyrlbam_Latnban_Latnbel_Cyrlbem_Latnben_Bengbho_Devabjn_Arabbjn_Latnbod_Tibtbos_Latnbug_Latnbul_Cyrlcat_Latnceb_Latnces_Latncjk_Latnckb_Arabcrh_Latncym_Latndan_Latndeu_Latndik_Latndyu_Latndzo_Tibtell_Grekeng_Latnepo_Latnest_Latneus_Latnewe_Latnfao_Latnpes_Arabfij_Latnfin_Latnfon_Latnfra_Latnfur_Latnfuv_Latngla_Latngle_Latnglg_Latngrn_Latnguj_Gujrhat_Latnhau_Latnheb_Hebrhin_Devahne_Devahrv_Latnhun_Latnhye_Armnibo_Latnilo_Latnind_Latnisl_Latnita_Latnjav_Latnjpn_Jpankab_Latnkac_Latnkam_Latnkan_Kndakas_Arabkas_Devakat_Georknc_Arabknc_Latnkaz_Cyrlkbp_Latnkea_Latnkhm_Khmrkik_Latnkin_Latnkir_Cyrlkmb_Latnkon_Latnkor_Hangkmr_Latnlao_Laoolvs_Latnlij_Latnlim_Latnlin_Latnlit_Latnlmo_Latnltg_Latnltz_Latnlua_Latnlug_Latnluo_Latnlus_Latnmag_Devamai_Devamal_Mlymmar_Devamin_Latnmkd_Cyrlplt_Latnmlt_Latnmni_Bengkhk_Cyrlmos_Latnmri_Latnzsm_Latnmya_Mymrnld_Latnnno_Latnnob_Latnnpi_Devanso_Latnnus_Latnnya_Latnoci_Latngaz_Latnory_Oryapag_Latnpan_Gurupap_Latnpol_Latnpor_Latnprs_Arabpbt_Arabquy_Latnron_Latnrun_Latnrus_Cyrlsag_Latnsan_Devasat_Bengscn_Latnshn_Mymrsin_Sinhslk_Latnslv_Latnsmo_Latnsna_Latnsnd_Arabsom_Latnsot_Latnspa_Latnals_Latnsrd_Latnsrp_Cyrlssw_Latnsun_Latnswe_Latnswh_Latnszl_Latntam_Tamltat_Cyrltel_Telutgk_Cyrltgl_Latntha_Thaitir_Ethitaq_Latntaq_Tfngtpi_Latntsn_Latntso_Latntuk_Latntum_Latntur_Latntwi_Latntzm_Tfnguig_Arabukr_Cyrlumb_Latnurd_Arabuzn_Latnvec_Latnvie_Latnwar_Latnwol_Latnxho_Latnydd_Hebryor_Latnyue_Hantzho_Hanszho_Hantzul_Latnc                       e Zd ZU dZeZddgZeZg Z	e
e   ed<   g Ze
e   ed<   	 	 	 	 	 	 	 	 	 	 	 	 	 d  fd	Zedefd	       Zedefd
       Zej(                  deddfd       Z	 d!de
e   dee
e      de
e   fdZ	 d!de
e   dee
e      de
e   fdZdedee   dee   fdZ	 	 	 d"de
e   dedee
e      dedef
 fdZd Zd Zd#dZdeddfdZd!dedee   dee   fdZ  xZ!S )$NllbTokenizerFasta  
    Construct a "fast" NLLB tokenizer (backed by HuggingFace's *tokenizers* library). Based on
    [BPE](https://huggingface.co/docs/tokenizers/python/latest/components.html?highlight=BPE#models).

    This tokenizer inherits from [`PreTrainedTokenizerFast`] which contains most of the main methods. Users should
    refer to this superclass for more information regarding those methods.

    The tokenization method is `<tokens> <eos> <language code>` for source language documents, and `<language code>
    <tokens> <eos>` for target language documents.

    Examples:

    ```python
    >>> from transformers import NllbTokenizerFast

    >>> tokenizer = NllbTokenizerFast.from_pretrained(
    ...     "facebook/nllb-200-distilled-600M", src_lang="eng_Latn", tgt_lang="fra_Latn"
    ... )
    >>> example_english_phrase = " UN Chief Says There Is No Military Solution in Syria"
    >>> expected_translation_french = "Le chef de l'ONU affirme qu'il n'y a pas de solution militaire en Syrie."
    >>> inputs = tokenizer(example_english_phrase, text_target=expected_translation_french, return_tensors="pt")
    ```

    Args:
        vocab_file (`str`):
            Path to the vocabulary file.
        bos_token (`str`, *optional*, defaults to `"<s>"`):
            The beginning of sequence token that was used during pretraining. Can be used a sequence classifier token.

            <Tip>

            When building a sequence using special tokens, this is not the token that is used for the beginning of
            sequence. The token used is the `cls_token`.

            </Tip>

        eos_token (`str`, *optional*, defaults to `"</s>"`):
            The end of sequence token.

            <Tip>

            When building a sequence using special tokens, this is not the token that is used for the end of sequence.
            The token used is the `sep_token`.

            </Tip>

        sep_token (`str`, *optional*, defaults to `"</s>"`):
            The separator token, which is used when building a sequence from multiple sequences, e.g. two sequences for
            sequence classification or for a text and a question for question answering. It is also used as the last
            token of a sequence built with special tokens.
        cls_token (`str`, *optional*, defaults to `"<s>"`):
            The classifier token which is used when doing sequence classification (classification of the whole sequence
            instead of per-token classification). It is the first token of the sequence when built with special tokens.
        unk_token (`str`, *optional*, defaults to `"<unk>"`):
            The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
            token instead.
        pad_token (`str`, *optional*, defaults to `"<pad>"`):
            The token used for padding, for example when batching sequences of different lengths.
        mask_token (`str`, *optional*, defaults to `"<mask>"`):
            The token used for masking values. This is the token used when training this model with masked language
            modeling. This is the token which the model will try to predict.
        tokenizer_file (`str`, *optional*):
            The path to a tokenizer file to use instead of the vocab file.
        src_lang (`str`, *optional*):
            The language to use as source language for translation.
        tgt_lang (`str`, *optional*):
            The language to use as target language for translation.
    	input_idsattention_maskprefix_tokenssuffix_tokensNc                 H   |t         }|| _        t        |	t              rt	        |	ddd      n|	}	|| _        t        |   d|||||||||
||	||d| |
|
nd| _        | j                  | j                        | _
        || _        | j                  | j                         y )NT)
normalizedlstripspecial)r   r   	bos_token	eos_token	sep_token	cls_token	unk_token	pad_tokensrc_langtgt_lang
mask_tokenadditional_special_tokenslegacy_behaviourr@    )FAIRSEQ_LANGUAGE_CODESr   
isinstancestrr	   r   super__init__	_src_langconvert_tokens_to_idscur_lang_coder   set_src_lang_special_tokens)selfr   r   r   r   r   r   r   r   r   r   r   r   r   kwargs	__class__s                  b/var/www/html/venv/lib/python3.12/site-packages/transformers/models/nllb/tokenization_nllb_fast.pyr   zNllbTokenizerFast.__init__w   s    " %,(>%$ *c* zd4N 	
 !1 	
!)!&?-	
 	
" &.%9z!77G ((8    returnc                 p    | j                   r)t        j                  j                  | j                         S dS )NF)r   ospathisfiler   s    r   can_save_slow_tokenizerz)NllbTokenizerFast.can_save_slow_tokenizer   s$    26//rww~~doo.LuLr   c                     | j                   S N)r   r  s    r   r   zNllbTokenizerFast.src_lang   s    ~~r   new_src_langc                 H    || _         | j                  | j                          y r  )r   r   )r   r	  s     r   r   zNllbTokenizerFast.src_lang   s    %((8r   token_ids_0token_ids_1c                 |    || j                   |z   | j                  z   S | j                   |z   |z   | j                  z   S )a  
        Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and
        adding special tokens. The special tokens depend on calling set_lang.

        An NLLB sequence has the following format, where `X` represents the sequence:

        - `input_ids` (for encoder) `X [eos, src_lang_code]`
        - `decoder_input_ids`: (for decoder) `X [eos, tgt_lang_code]`

        BOS is never used. Pairs of sequences are not the expected use case, but they will be handled without a
        separator.

        Args:
            token_ids_0 (`List[int]`):
                List of IDs to which the special tokens will be added.
            token_ids_1 (`List[int]`, *optional*):
                Optional second list of IDs for sequence pairs.

        Returns:
            `List[int]`: list of [input IDs](../glossary#input-ids) with the appropriate special tokens.
        )r   r   )r   r  r  s      r    build_inputs_with_special_tokensz2NllbTokenizerFast.build_inputs_with_special_tokens   sG    0 %%3d6H6HHH!!K/+=@R@RRRr   c                     | j                   g}| j                  g}|t        ||z   |z         dgz  S t        ||z   |z   |z   |z   |z         dgz  S )a  
        Create a mask from the two sequences passed to be used in a sequence-pair classification task. nllb does not
        make use of token type ids, therefore a list of zeros is returned.

        Args:
            token_ids_0 (`List[int]`):
                List of IDs.
            token_ids_1 (`List[int]`, *optional*):
                Optional second list of IDs for sequence pairs.

        Returns:
            `List[int]`: List of zeros.

        r   )sep_token_idcls_token_idlen)r   r  r  sepclss        r   $create_token_type_ids_from_sequencesz6NllbTokenizerFast.create_token_type_ids_from_sequences   sm    $   !  !s[(3./1#553$s*S0;>DEKKr   return_tensorsr   r   c                 v    ||t        d      || _         | |fd|d|}| j                  |      }||d<   |S )zIUsed by translation pipeline, to prepare inputs for the generate functionzATranslation requires a `src_lang` and a `tgt_lang` for this modelT)add_special_tokensr  forced_bos_token_id)
ValueErrorr   r   )r   
raw_inputsr  r   r   extra_kwargsinputstgt_lang_ids           r   _build_translation_inputsz+NllbTokenizerFast._build_translation_inputs   sY     x/`aa jiT.i\hi00:(3$%r   	src_texts	tgt_textsc                 B    || _         || _        t        |   ||fi |S r  )r   r   r   prepare_seq2seq_batch)r   r   r   r!  r   r   r   s         r   r#  z'NllbTokenizerFast.prepare_seq2seq_batch   s*     ! w,Y	LVLLr   c                 8    | j                  | j                        S r  )r   r   r  s    r   _switch_to_input_modez'NllbTokenizerFast._switch_to_input_mode      //>>r   c                 8    | j                  | j                        S r  )set_tgt_lang_special_tokensr   r  s    r   _switch_to_target_modez(NllbTokenizerFast._switch_to_target_mode  r&  r   c                    | j                  |      | _        | j                  r%g | _        | j                  | j                  g| _        n$| j                  g| _        | j                  g| _        | j                  | j                        }| j                  | j
                        }t        j                  |dgz   |z   |ddgz   |z   t        t        ||z   | j                  | j
                  z                     | j                  _        y)zReset the special tokens to the source lang setting.
        - In legacy mode: No prefix and suffix=[eos, src_lang_code].
        - In default mode: Prefix=[src_lang_code], suffix = [eos]
        $A$Bsinglepairspecial_tokensNr   r   r   r   eos_token_idr   convert_ids_to_tokensr   TemplateProcessinglistzip
_tokenizerpost_processor)r   r   prefix_tokens_strsuffix_tokens_strs       r   r   z-NllbTokenizerFast.set_src_lang_special_tokens
  s    
 "77A  !#D"&"3"3T5G5G!HD"&"4"4!5D"&"3"3!4D 66t7I7IJ 66t7I7IJ)3)F)F$v-0AA"dD\14EE$58I$I4K]K]`d`r`rKr st*
&r   langc                    | j                  |      | _        | j                  r%g | _        | j                  | j                  g| _        n$| j                  g| _        | j                  g| _        | j                  | j                        }| j                  | j
                        }t        j                  |dgz   |z   |ddgz   |z   t        t        ||z   | j                  | j
                  z                     | j                  _        y)zReset the special tokens to the target lang setting.
        - In legacy mode: No prefix and suffix=[eos, tgt_lang_code].
        - In default mode: Prefix=[tgt_lang_code], suffix = [eos]
        r+  r,  r-  Nr1  )r   r;  r9  r:  s       r   r(  z-NllbTokenizerFast.set_tgt_lang_special_tokens!  s    
 "77=  !#D"&"3"3T5G5G!HD"&"4"4!5D"&"3"3!4D 66t7I7IJ 66t7I7IJ)3)F)F$v-0AA"dD\14EE$58I$I4K]K]`d`r`rKr st*
&r   save_directoryfilename_prefixc                    | j                   st        d      t        j                  j	                  |      st
        j                  d| d       y t        j                  j                  ||r|dz   ndt        d   z         }t        j                  j                  | j                        t        j                  j                  |      k7  rt        | j                  |       |fS )NzhYour fast tokenizer does not have the necessary information to save the vocabulary for a slow tokenizer.zVocabulary path (z) should be a directory.- r   )r  r  r  r  isdirloggererrorjoinVOCAB_FILES_NAMESabspathr   r   )r   r=  r>  out_vocab_files       r   save_vocabularyz!NllbTokenizerFast.save_vocabulary7  s    ++ 
 ww}}^,LL,^,<<TUVo_s22QbcoQpp
 77??4??+rww~/NNT__n5  r   )NN<s></s>rK  rJ  z<unk>z<pad>z<mask>NNNFr  )r@   NrJ   )r   N)"__name__
__module____qualname____doc__rF  vocab_files_namesmodel_input_namesr   slow_tokenizer_classr   r   int__annotations__r   r   propertyboolr  r   r   setterr   r  r  r  r
   r#  r%  r)  r   r(  r   rI  __classcell__)r   s   @r   r   r   *   s   CJ *$&67(!M49!!M49! "&09d M M M #   __9S 9T 9 9
 JNS9S3;DI3FS	cS< JNL9L3;DI3FL	cL2
*-
9A#
RZ[^R_
 #)-"
M9
M 
M DI&	
M
 
M 

M??
.
 
 
,!c !HSM !]bcf]g !r   r   )r  shutilr   typingr   r   r   
tokenizersr   tokenization_utilsr	   r
   tokenization_utils_fastr   utilsr   r   tokenization_nllbr   
get_loggerrL  rC  rF  r   r   r   r   r   <module>ra     si     
  ( ( ! ; > 8 0M 
		H	% $=P`a  R& ^!/ ^!r   