sgD>ddlZddlmZddlmZmZmZddlmZddl m Z m Z ddl m Z ddlmZmZerd d lmZndZej&eZd d d ZgdZGdde Zy)N)copyfile)ListOptionalTuple) processors) AddedToken BatchEncoding)PreTrainedTokenizerFast)is_sentencepiece_availablelogging) NllbTokenizerzsentencepiece.bpe.modelztokenizer.json) vocab_filetokenizer_file)ace_Arabace_Latnacm_Arabacq_Arabaeb_Arabafr_Latnajp_Arabaka_Latnamh_Ethiapc_Arabarb_Arabars_Arabary_Arabarz_Arabasm_Bengast_Latnawa_Devaayr_Latnazb_Arabazj_Latnbak_Cyrlbam_Latnban_Latnbel_Cyrlbem_Latnben_Bengbho_Devabjn_Arabbjn_Latnbod_Tibtbos_Latnbug_Latnbul_Cyrlcat_Latnceb_Latnces_Latncjk_Latnckb_Arabcrh_Latncym_Latndan_Latndeu_Latndik_Latndyu_Latndzo_Tibtell_Grekeng_Latnepo_Latnest_Latneus_Latnewe_Latnfao_Latnpes_Arabfij_Latnfin_Latnfon_Latnfra_Latnfur_Latnfuv_Latngla_Latngle_Latnglg_Latngrn_Latnguj_Gujrhat_Latnhau_Latnheb_Hebrhin_Devahne_Devahrv_Latnhun_Latnhye_Armnibo_Latnilo_Latnind_Latnisl_Latnita_Latnjav_Latnjpn_Jpankab_Latnkac_Latnkam_Latnkan_Kndakas_Arabkas_Devakat_Georknc_Arabknc_Latnkaz_Cyrlkbp_Latnkea_Latnkhm_Khmrkik_Latnkin_Latnkir_Cyrlkmb_Latnkon_Latnkor_Hangkmr_Latnlao_Laoolvs_Latnlij_Latnlim_Latnlin_Latnlit_Latnlmo_Latnltg_Latnltz_Latnlua_Latnlug_Latnluo_Latnlus_Latnmag_Devamai_Devamal_Mlymmar_Devamin_Latnmkd_Cyrlplt_Latnmlt_Latnmni_Bengkhk_Cyrlmos_Latnmri_Latnzsm_Latnmya_Mymrnld_Latnnno_Latnnob_Latnnpi_Devanso_Latnnus_Latnnya_Latnoci_Latngaz_Latnory_Oryapag_Latnpan_Gurupap_Latnpol_Latnpor_Latnprs_Arabpbt_Arabquy_Latnron_Latnrun_Latnrus_Cyrlsag_Latnsan_Devasat_Bengscn_Latnshn_Mymrsin_Sinhslk_Latnslv_Latnsmo_Latnsna_Latnsnd_Arabsom_Latnsot_Latnspa_Latnals_Latnsrd_Latnsrp_Cyrlssw_Latnsun_Latnswe_Latnswh_Latnszl_Latntam_Tamltat_Cyrltel_Telutgk_Cyrltgl_Latntha_Thaitir_Ethitaq_Latntaq_Tfngtpi_Latntsn_Latntso_Latntuk_Latntum_Latntur_Latntwi_Latntzm_Tfnguig_Arabukr_Cyrlumb_Latnurd_Arabuzn_Latnvec_Latnvie_Latnwar_Latnwol_Latnxho_Latnydd_Hebryor_Latnyue_Hantzho_Hanszho_Hantzul_Latnc eZdZUdZeZddgZeZgZ e e e d<gZ e e e d< d fd Zedefd Zedefd Zej(d eddfd Z d!d e e dee e de e fdZ d!d e e dee e de e fdZdedeedeefdZ d"de ededee ededef fd ZdZdZd#dZdeddfdZd!dedeedeefdZ xZ!S)$NllbTokenizerFasta Construct a "fast" NLLB tokenizer (backed by HuggingFace's *tokenizers* library). Based on [BPE](https://huggingface.co/docs/tokenizers/python/latest/components.html?highlight=BPE#models). This tokenizer inherits from [`PreTrainedTokenizerFast`] which contains most of the main methods. Users should refer to this superclass for more information regarding those methods. The tokenization method is ` ` for source language documents, and ` ` for target language documents. Examples: ```python >>> from transformers import NllbTokenizerFast >>> tokenizer = NllbTokenizerFast.from_pretrained( ... "facebook/nllb-200-distilled-600M", src_lang="eng_Latn", tgt_lang="fra_Latn" ... ) >>> example_english_phrase = " UN Chief Says There Is No Military Solution in Syria" >>> expected_translation_french = "Le chef de l'ONU affirme qu'il n'y a pas de solution militaire en Syrie." >>> inputs = tokenizer(example_english_phrase, text_target=expected_translation_french, return_tensors="pt") ``` Args: vocab_file (`str`): Path to the vocabulary file. bos_token (`str`, *optional*, defaults to `""`): The beginning of sequence token that was used during pretraining. Can be used a sequence classifier token. When building a sequence using special tokens, this is not the token that is used for the beginning of sequence. The token used is the `cls_token`. eos_token (`str`, *optional*, defaults to `""`): The end of sequence token. When building a sequence using special tokens, this is not the token that is used for the end of sequence. The token used is the `sep_token`. sep_token (`str`, *optional*, defaults to `""`): The separator token, which is used when building a sequence from multiple sequences, e.g. two sequences for sequence classification or for a text and a question for question answering. It is also used as the last token of a sequence built with special tokens. cls_token (`str`, *optional*, defaults to `""`): The classifier token which is used when doing sequence classification (classification of the whole sequence instead of per-token classification). It is the first token of the sequence when built with special tokens. unk_token (`str`, *optional*, defaults to `""`): The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this token instead. pad_token (`str`, *optional*, defaults to `""`): The token used for padding, for example when batching sequences of different lengths. mask_token (`str`, *optional*, defaults to `""`): The token used for masking values. This is the token used when training this model with masked language modeling. This is the token which the model will try to predict. tokenizer_file (`str`, *optional*): The path to a tokenizer file to use instead of the vocab file. src_lang (`str`, *optional*): The language to use as source language for translation. tgt_lang (`str`, *optional*): The language to use as target language for translation. input_idsattention_mask prefix_tokens suffix_tokensNc H| t} ||_t| trt | dddn| } | |_t |d||||||||| | | | | d || | nd|_|j|j|_ | |_ |j|jy)NT) normalizedlstripspecial) rr bos_token eos_token sep_token cls_token unk_token pad_tokensrc_langtgt_lang mask_tokenadditional_special_tokenslegacy_behaviourr@) FAIRSEQ_LANGUAGE_CODESr isinstancestrr rsuper__init__ _src_langconvert_tokens_to_ids cur_lang_coderset_src_lang_special_tokens)selfrrrrrrrrrrrrrkwargs __class__s b/var/www/html/venv/lib/python3.12/site-packages/transformers/models/nllb/tokenization_nllb_fast.pyrzNllbTokenizerFast.__init__ws" % ,(> %$*c* zd4 N  !1  !)!&?-  "&.%9z!77G   ((8returncp|jr)tjj|jSdS)NF)rospathisfilers rcan_save_slow_tokenizerz)NllbTokenizerFast.can_save_slow_tokenizers$26//rww~~doo.LuLrc|jSN)rrs rrzNllbTokenizerFast.src_langs ~~r new_src_langcH||_|j|jyr)rr)rr s rrzNllbTokenizerFast.src_langs% ((8r token_ids_0 token_ids_1c|||j|z|jzS|j|z|z|jzS)a Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and adding special tokens. The special tokens depend on calling set_lang. An NLLB sequence has the following format, where `X` represents the sequence: - `input_ids` (for encoder) `X [eos, src_lang_code]` - `decoder_input_ids`: (for decoder) `X [eos, tgt_lang_code]` BOS is never used. Pairs of sequences are not the expected use case, but they will be handled without a separator. Args: token_ids_0 (`List[int]`): List of IDs to which the special tokens will be added. token_ids_1 (`List[int]`, *optional*): Optional second list of IDs for sequence pairs. Returns: `List[int]`: list of [input IDs](../glossary#input-ids) with the appropriate special tokens. )rr)rr r s r build_inputs_with_special_tokensz2NllbTokenizerFast.build_inputs_with_special_tokenssG0  %% 3d6H6HH H!!K/+=@R@RRRrc|jg}|jg}|t||z|zdgzSt||z|z|z|z|zdgzS)a Create a mask from the two sequences passed to be used in a sequence-pair classification task. nllb does not make use of token type ids, therefore a list of zeros is returned. Args: token_ids_0 (`List[int]`): List of IDs. token_ids_1 (`List[int]`, *optional*): Optional second list of IDs for sequence pairs. Returns: `List[int]`: List of zeros. r) sep_token_id cls_token_idlen)rr r sepclss r$create_token_type_ids_from_sequencesz6NllbTokenizerFast.create_token_type_ids_from_sequencessm$  !  !  s[(3./1#5 53$s*S0;>DEKKrreturn_tensorsrrc v|| td||_||fd|d|}|j|}||d<|S)zIUsed by translation pipeline, to prepare inputs for the generate functionzATranslation requires a `src_lang` and a `tgt_lang` for this modelT)add_special_tokensrforced_bos_token_id) ValueErrorrr)r raw_inputsrrr extra_kwargsinputs tgt_lang_ids r_build_translation_inputsz+NllbTokenizerFast._build_translation_inputssY  x/`a a  jiT.i\hi00: (3$% r src_texts tgt_textsc B||_||_t| ||fi|Sr)rrrprepare_seq2seq_batch)rr rr!rrrs rr#z'NllbTokenizerFast.prepare_seq2seq_batchs*!   w,Y LVLLrc8|j|jSr)rrrs r_switch_to_input_modez'NllbTokenizerFast._switch_to_input_mode// >>rc8|j|jSr)set_tgt_lang_special_tokensrrs r_switch_to_target_modez(NllbTokenizerFast._switch_to_target_moder&rc  |j||_|jr%g|_|j|jg|_n$|jg|_|jg|_|j |j}|j |j }tj|dgz|z|ddgz|ztt||z|j|j z|j_ y)zReset the special tokens to the source lang setting. - In legacy mode: No prefix and suffix=[eos, src_lang_code]. - In default mode: Prefix=[src_lang_code], suffix = [eos] $A$Bsinglepairspecial_tokensN rrrr eos_token_idrconvert_ids_to_tokensrTemplateProcessinglistzip _tokenizerpost_processor)rrprefix_tokens_strsuffix_tokens_strs rrz-NllbTokenizerFast.set_src_lang_special_tokens s "77A  !#D "&"3"3T5G5G!HD "&"4"4!5D "&"3"3!4D  66t7I7IJ 66t7I7IJ)3)F)F$v-0AA"dD\14EE$58I$I4K]K]`d`r`rKr st* &rlangc  |j||_|jr%g|_|j|jg|_n$|jg|_|jg|_|j |j}|j |j }tj|dgz|z|ddgz|ztt||z|j|j z|j_ y)zReset the special tokens to the target lang setting. - In legacy mode: No prefix and suffix=[eos, tgt_lang_code]. - In default mode: Prefix=[tgt_lang_code], suffix = [eos] r+r,r-Nr1)rr;r9r:s rr(z-NllbTokenizerFast.set_tgt_lang_special_tokens!s "77=  !#D "&"3"3T5G5G!HD "&"4"4!5D "&"3"3!4D  66t7I7IJ 66t7I7IJ)3)F)F$v-0AA"dD\14EE$58I$I4K]K]`d`r`rKr st* &rsave_directoryfilename_prefixc|js tdtjj |st j d|dytjj||r|dzndtdz}tjj|jtjj|k7rt|j||fS)NzhYour fast tokenizer does not have the necessary information to save the vocabulary for a slow tokenizer.zVocabulary path (z) should be a directory.-r) rrrrisdirloggererrorjoinVOCAB_FILES_NAMESabspathrr)rr=r>out_vocab_files rsave_vocabularyz!NllbTokenizerFast.save_vocabulary7s++  ww}}^, LL,^,<rKrJzzzNNNFr)r@NrJ)rN)"__name__ __module__ __qualname____doc__rFvocab_files_namesmodel_input_namesrslow_tokenizer_classrrint__annotations__rrpropertyboolrrrsetterrrrrr r#r%r)rr(rrI __classcell__)rs@rrr*sCJ*$&67(!M49!!M49!"&09dMMM#__9S9T99 JNS9S3;DI3FS cS<JNL9L3;DI3FL cL2 *- 9A# RZ[^R_ #)-" M9 M MDI& M  M  M?? .   ,!c!HSM!]bcf]g!rr)rshutilrtypingrrr tokenizersrtokenization_utilsr r tokenization_utils_fastr utilsr r tokenization_nllbr get_loggerrLrCrFrrrrrrasi  ((!;>80M   H %$=P`aR&^!/^!r