
    sgw                         d Z ddlZddlZddlZddlZddlmZ ddlmZm	Z	m
Z
 ddlZddlmZmZmZ ddlmZ ddlmZ dd	lmZ d
dlmZmZ d
dlmZmZmZmZmZ  ej@                  e!      Z"dddddZ# G d de      Z$y)z!Tokenization classes for Whisper.    N)	lru_cache)ListOptionalTuple)
AddedTokenpre_tokenizers
processors   )BatchEncoding)PreTrainedTokenizerFast)logging   )BasicTextNormalizerEnglishTextNormalizer)	LANGUAGESTASK_IDSTO_LANGUAGE_CODEWhisperTokenizer_decode_asrz
vocab.jsonztokenizer.jsonz
merges.txtznormalizer.json)
vocab_filetokenizer_filemerges_filenormalizer_filec                       e Zd ZdZeZddgZeZ	 	 	 	 	 	 	 	 	 	 	 d3 fd	Z	de
f fdZde
f fdZ	 d4defd	Zd5d
Zed6d       Zd7defdZd Z	 	 	 	 	 	 	 	 d8dededededededededef fdZdddddedededef fdZd Zd7dZd Zed7d       Zd9dedee   dee   fdZd:d ed!ed"efd#Z e!de"e#   fd$       Z$d9de"e#   fd%Z%	 d;d&e"e#   d'ee"e#      d(ede"e#   f fd)Z&d<d*Z'd+ Z(d=d,efd-Z)d.e"e#   d/e#d0e#fd1Z*ed2        Z+ xZ,S )>WhisperTokenizerFastag	  
    Construct a "fast" Whisper tokenizer (backed by HuggingFace's *tokenizers* library).

    This tokenizer inherits from [`PreTrainedTokenizerFast`] which contains most of the main methods. Users should
    refer to this superclass for more information regarding those methods.

    Args:
        vocab_file (`str`, *optional*):
            Path to the vocabulary file.
        merges_file (`str`, *optional*):
            Path to the merges file.
        normalizer_file (`str`, *optional*):
            Path to the normalizer_file file.
        tokenizer_file (`str`, *optional*):
            Path to [tokenizers](https://github.com/huggingface/tokenizers) file (generally has a .json extension) that
            contains everything needed to load the tokenizer.
        unk_token (`str`, *optional*, defaults to `"<|endoftext|>"`):
            The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
            token instead.
        bos_token (`str`, *optional*, defaults to `"<|endoftext|>"`):
            The beginning of sequence token. The `decoder_start_token_id` is used to set the first token as
            `"<|startoftranscript|>"` when generating.
        eos_token (`str`, *optional*, defaults to `"<|endoftext|>"`):
            The end of sequence token.
        add_prefix_space (`bool`, *optional*, defaults to `False`):
            Whether or not to add an initial space to the input. This allows to treat the leading word just as any
            other word. (Whisper tokenizer detect beginning of words by the preceding space).
        language (`str`, *optional*):
            The language of the transcription text. The corresponding language id token is appended to the start of the
            sequence for multilingual speech recognition and speech translation tasks, e.g. for Spanish the token
            `"<|es|>"` is appended to the start of sequence. This should be used for multilingual fine-tuning only.
        task (`str`, *optional*):
            Task identifier to append at the start of sequence (if any). This should be used for mulitlingual
            fine-tuning, with `"transcribe"` for speech recognition and `"translate"` for speech translation.
        predict_timestamps (`bool`, *optional*, defaults to `False`):
            Whether to omit the `<|notimestamps|>` token at the start of the sequence.
    	input_idsattention_maskFc           	      4   t        |t              rt        |dddd      n|}t        |t              rt        |dddd      n|}t        |t              rt        |dddd      n|}t        |   ||f|||||d| |j                  dd      | _        t        j                  | j                  j                  j                               }|j                  d|      |k7  r;t        t        |j                  d            }||d<    |di || j                  _
        |1t        |d	      5 }t        j                   |      | _        d d d        nd | _        || _        t'        j(                  d
      | _        |	| _        |
| _        || _        y # 1 sw Y   @xY w)NFT)lstriprstrip
normalizedspecial)r   	unk_token	bos_token	eos_tokenadd_prefix_spaceadd_bos_tokenr&   typeutf-8encodingz<\|(\d+\.\d+)\|> )
isinstancestrr   super__init__popr'   jsonloadsbackend_tokenizerpre_tokenizer__getstate__getgetattrr   openloadenglish_spelling_normalizerr&   recompiletimestamp_patlanguagetaskpredict_timestamps)selfr   r   r   r   r#   r$   r%   r&   r?   r@   rA   kwargspre_tok_statepre_tok_classvocab_handle	__class__s                   h/var/www/html/venv/lib/python3.12/site-packages/transformers/models/whisper/tokenization_whisper_fast.pyr0   zWhisperTokenizerFast.__init__W   s   " )S) yuX\] 	 )S) yuX\] 	 )S) yuX\] 	 			
 *-		
 		
 $ZZ?

4#9#9#G#G#T#T#VW/1ABFVV#NM4E4Ef4MNM0@M,-3@3Q=3QD""0&o8 KL3799\3J0K K 04D, 0ZZ(;< 	"4K Ks   ,FFreturnc                     |j                  dd      }| j                  s!|rJ d| j                  j                   d       t	        |   |i |S Nis_split_into_wordsFzYou need to instantiate z? with add_prefix_space=True to use it with pretokenized inputs.)r7   r&   rG   __name__r/   _batch_encode_plusrB   argsrC   rL   rG   s       rH   rN   z'WhisperTokenizerFast._batch_encode_plus   sb    $jj)>F$$,? 	
&t~~'>'>&? @2 2	
?
 w)4:6::    c                     |j                  dd      }| j                  s!|rJ d| j                  j                   d       t	        |   |i |S rK   )r7   r&   rG   rM   r/   _encode_plusrO   s       rH   rS   z!WhisperTokenizerFast._encode_plus   sb    $jj)>F$$,? 	
&t~~'>'>&? @2 2	
?
 w#T4V44rQ   c                    | j                   d   dz   }g g}d}d}d}	t        |      D ]  \  }
}||k\  r|t        ||z
  |z        }||k  r7|
dk\  xr ||
dz
     |k\  xr ||
dz
     |k\   }|r	|||z  z  }n|	}||	z  }|dd }|}	|}|j                  d||z   dd	       |j                  g        |d   j                  |        |D cg c]'  }t	        |t
              r|n| j                  ||
      ) }}dj                  |      S c c}w )z
        Timestamp tokens are above the special tokens' id range and are ignored by `decode()`. This method decodes
        given tokens with timestamps tokens annotated, e.g. "<|1.08|>".
        r   g           Nz<|z.2fz|>skip_special_tokens )all_special_ids	enumeratefloatappendr-   r.   decodejoin)rB   	token_idsrY   time_precisionsegment_sizetimestamp_beginoutputscur_max_timestampprev_segments_lenpenultimate_timestampitoken	timestamplast_was_single_endingss                  rH   _decode_with_timestampsz,WhisperTokenizerFast._decode_with_timestamps   sr    ..r2Q6$ #!), 	*HAu'!5?#:n"LM	00-.!V .!!a%(O;c	!a%@PTc@c9* .)^l-JJ),A))-BB)")#2,(9%$-!Y1B%BC#HKLr"""5)-	*0 kr
efAs#AQL_)``
 
 www
s   ,Dc                    g }dt        t        |            v r1t        |d      r%t        |j                        r|j	                         }t        j                  |      }|j                  d   dkD  r#t        |j                        dkD  rt        d      | j                  d   dz   }||k\  }t        j                  |dd |dd z        d   dz   }|j                  d   dk(  r|j                         dk  rg S t        j                  |      d   d   dz   |vr2t        j                  |t        j                  |      d   d   dz         }t        j                  |      d   d   }d}	d}
|D ]  }||| }t        |      dkD  r|d   j                         |z
  }|d   j                         |z
  }||	k  r-|dk\  xr ||dz
     |k\  xr ||dz
     |k\   }|r|
|z  }
n|
|	z  }
|}	| j                  |      }| j!                  |      }| j#                  |      }|j                  |||z  |
|z  z   ||z  |
|z  z   fd	       |} |S )
a  
        Compute offsets for a given tokenized input

        Args:
            token_ids (`Union[int, List[int], np.ndarray, torch.Tensor, tf.Tensor]`):
                List of tokenized input ids. Can be obtained using the `__call__` method.
            time_precision (`float`, *optional*, defaults to 0.02):
                The time ratio to convert from token to time.
            segment_size (`int`, *optional*, defaults to 1500):
                The number of features in the input mel spectrogram.
        torchcpur   r   z)Can only process a single input at a timerU   NrV   )textrk   )r.   r(   hasattrcallablerq   nparrayshapelen
ValueErrorr[   wheresumr^   item_preprocess_token_ids_decode_filter_timestamp_ids)rB   ra   rb   rc   offsetsrd   timestamp_tokensconsecutive
last_slicerf   rg   current_slicesliced_tokensstart_timestamp_positionend_timestamp_positionis_single_endingrr   s                    rH   _compute_offsetsz%WhisperTokenizerFast._compute_offsets   s    c$y/**	50IhW`WdWdNe!IHHY'	??1!c)//&:Q&>HII..r2Q6$7hh/47G7KKLQORSSQ1$)9)=)=)?1)DIXX&'*2.2+E))K:J1KA1Nr1RUV1VWKXX./215
( 	'M%j?M=!A%+8+;+@+@+B_+T()6r):)?)?)AO)S&+.??'1Q (!*q.1_DuS]`aSaIbfuIu<$ ()\9))->>)$:! !% : := I||M211$7 $4~EHY\jHjj2^CFWZhFhh& 'J?	'B rQ   c                 h    | j                  t        d      D cg c]
  }d||z  z   c}      S c c}w )a  
        Compute the timestamp token ids for a given precision and save to least-recently used (LRU) cache.

        Args:
            time_precision (`float`, *optional*, defaults to 0.02):
                The time ratio to convert from token to time.
        i  z<|%.2f|>)convert_tokens_to_idsrange)rB   rb   ri   s      rH   timestamp_idsz"WhisperTokenizerFast.timestamp_ids  s5     ))X]^fXg*hSTJ!n:L,M*hii*hs   /rY   c                 t    |r5| j                  d      }| j                  d      }| j                  |||      }|S )a  
        Pre-process the token ids for decoding by removing the prompt tokens ids and timestamp token ids.

        Args:
            token_ids (`Union[int, List[int], np.ndarray, torch.Tensor, tf.Tensor]`):
                List of tokenized input ids. Typically, obtained using the `__call__` method of the tokenizer.
            skip_special_tokens (`bool`, *optional*, defaults to `False`):
                Whether or not to remove special tokens from the token ids. If `True`, the prompt token ids will be
                removed.
        <|startofprev|><|startoftranscript|>)r   _strip_prompt)rB   ra   rY   prompt_token_iddecoder_start_token_ids        rH   r}   z*WhisperTokenizerFast._preprocess_token_ids(  sE     "889JKO%)%?%?@W%X"**9oG]^IrQ   c                 D    t        j                  | j                  d|      S )NrZ   )r<   subr>   )rB   ra   s     rH   r   z*WhisperTokenizerFast._filter_timestamp_ids;  s    vvd(("i88rQ   clean_up_tokenization_spacesoutput_offsetsrb   decode_with_timestamps	normalizebasic_normalizeremove_diacriticsc
           	          | j                  ||      }t        |   |f|||||	d|
}|r| j                  |||      }n| j	                  |      }|r| j                  ||      }||dS |S )a	  
        Converts a sequence of ids in a string, using the tokenizer and vocabulary with options to remove special
        tokens and clean up tokenization spaces.

        Similar to doing `self.convert_tokens_to_string(self.convert_ids_to_tokens(token_ids))`.

        Args:
            token_ids (`Union[int, List[int], np.ndarray, torch.Tensor, tf.Tensor]`):
                List of tokenized input ids. Can be obtained using the `__call__` method.
            skip_special_tokens (`bool`, *optional*, defaults to `False`):
                Whether or not to remove special tokens in the decoding. Will remove the previous tokens (pre-prompt)
                if present.
            clean_up_tokenization_spaces (`bool`, *optional*):
                Whether or not to clean up the tokenization spaces. If `None`, will default to
                `self.clean_up_tokenization_spaces` (available in the `tokenizer_config`).
            output_offsets (`bool`, *optional*, defaults to `False`):
                Whether or not to output the offsets of the tokens. This should only be set if the model predicted
                timestamps. If there are previous tokens (pre-prompt) to decode, they will only appear in the decoded
                text if they contain timestamp tokens.
            time_precision (`float`, *optional*, defaults to 0.02):
                The time ratio to convert from token to time.
            decode_with_timestamps (`bool`, *optional*, defaults to `False`):
                Whether or not to decode with timestamps included in the raw text.
            normalize (`bool`, *optional*, defaults to `False`):
                Whether or not to apply the English text normalizer to the decoded text. Only applicable when the
                target text is in English. Otherwise, the basic text normalizer should be applied.
            basic_normalize (`bool`, *optional*, defaults to `False`):
                Whether or not to apply the Basic text normalizer to the decoded text. Applicable to multilingual
                target text.
            remove_diacritics (`bool`, *optional*, defaults to `False`):
                Whether or not to remove diacritics when applying the Basic text normalizer. Removing diacritics may
                destroy information in the decoded text, hence it should be used with caution.
            kwargs (additional keyword arguments, *optional*):
                Will be passed to the underlying model specific decode method.
        Returns:
            `str`: The decoded sentence.
        rX   )rY   r   r   r   r   )rb   rY   )rb   )rr   r   )r}   r/   r_   rn   r   r   )rB   ra   rY   r   r   rb   r   r   r   r   rC   filtered_idsrr   r   rG   s                 rH   r_   zWhisperTokenizerFast.decode?  s    d 11 3 2 

 w~
 3)E+/
 
 "//^Qd 0 D --d3D ++In+UG W55rQ   )r   r   r   c                ~    t        |   |i |}|r| j                  |      }|S |r| j                  ||      }|S |S )Nr   )r/   r~   
_normalize_basic_normalize)	rB   r   r   r   rP   rC   rr   
clean_textrG   s	           rH   r~   zWhisperTokenizerFast._decode  sS     w//.J..tGX.YJKrQ   c                 N    t        j                  d       | j                  |      S )NzThe private method `_normalize` is deprecated and will be removed in v5 of Transformers.You can normalize an input string using the Whisper English normalizer using the `normalize` method.)warningswarnr   )rB   rr   s     rH   r   zWhisperTokenizerFast._normalize  s%    s	
 ~~d##rQ   c                 R    t        j                  d       | j                  ||      S )NzThe private method `_basic_normalize` is deprecated and will be removed in v5 of Transformers.You can normalize an input string using the Whisper basic normalizer using the `basic_normalize` method.r   )r   r   r   )rB   rr   r   s      rH   r   z%WhisperTokenizerFast._basic_normalize  s-    w	
 ##D<M#NNrQ   c                 <    t        | j                        } ||      S )z
        Normalize a given string using the `EnglishTextNormalizer` class, which preforms commons transformation on
        english text.
        )r   r;   )rB   rr   
normalizers      rH   r   zWhisperTokenizerFast.normalize  s    
 +4+K+KL
$rQ   c                 *    t        |      } ||       S )z
        Normalize a given string using the `BasicTextNormalizer` class, which preforms commons transformation on
        multilingual text.
        r   )r   )rr   r   r   s      rH   r   z$WhisperTokenizerFast.basic_normalize  s     );LM
$rQ   save_directoryfilename_prefixc           	         | j                   j                  j                  ||      }t        j                  j                  ||r|dz   ndt        d   z         }| j                  Lt        |dd      5 }|j                  t        j                  | j                  dd	d
      dz          d d d        t        |      |fz   S # 1 sw Y   xY w)N)name-rZ   r   wr)   r*   rV   TF)indent	sort_keysensure_ascii
)
_tokenizermodelsaveospathr`   VOCAB_FILES_NAMESr;   r9   writer2   dumpstuple)rB   r   r   filesr   fs         rH   save_vocabularyz$WhisperTokenizerFast.save_vocabulary  s    %%**>*P'',,o_s22QbctQuu
 ++7osW= JJt??UYhmnquu
 U|000 s   36C  C	r?   r@   rA   c           	         ||n| j                   | _         ||n| j                  | _        ||n| j                  | _        | j                  }| j	                  |      }| j
                  }| j                  }dj                  |D cg c]  }| d	 c}      }	t        j                  |	 d| d|	 d| d||fgt        ||            | j                  _        yc c}w )a  
        Override the prefix tokens appended to the start of the label sequence. This method can be used standalone to
        update the prefix tokens as required when fine-tuning. Example:

        ```python
        >>> # instantiate the tokenizer and set the prefix token to Spanish
        >>> tokenizer = WhisperTokenizerFast.from_pretrained("openai/whisper-tiny", language="spanish")
        >>> # now switch the prefix token from Spanish to French
        >>> tokenizer.set_prefix_tokens(language="french")
        ```

        Args:
            language (`str`, *optional*, defaults to `None`):
                The language of the transcription text.
            task (`str`, *optional*, defaults to `None`):
                Task identifier to append at the start of sequence (if any).
            predict_timestamps (`bool`, *optional*, defaults to `None`):
                Whether to omit the `<|notimestamps|>` token at the start of the sequence.
        N z:0z $A:0 z $A:0 $B:1 z:1)singlepairspecial_tokens)r?   r@   rA   prefix_tokensconvert_ids_to_tokensr%   eos_token_idr`   r	   TemplateProcessingzipr4   post_processor)
rB   r?   r@   rA   prefix_token_idsprefixeseosr   rj   prefix_templates
             rH   set_prefix_tokensz&WhisperTokenizerFast.set_prefix_tokens  s    ( %-$8dmm ,D$))	8J8V"4\`\s\s----.>?nn((((h#GUugRL#GH0:0M0M%&fSE4#$KuB7l#X/01
- $Hs   Cc           	         | j                  d      }| j                  d      }| j                  d      }| j                  d      }t        t        j                               }| j                  | j                  j                         | _        | j                  t        v rt        | j                     }n| j                  t        j                         v r| j                  }nnt        | j                        dk(  }t        d| j                   d|rt        t        j                               nt        t        j                                d      | j                  1| j                  t        vrt        d	| j                   d
t               |g}| j                  &|j                  |dz   |j                        z          | j                  "|j                  | j                  dk(  r|n|       | j                  s|j                  |       |S )Nr   z<|translate|>z<|transcribe|>z<|notimestamps|>rV   zUnsupported language: z. Language should be one of: .zUnsupported task: z. Task should be in: r   
transcribe)r   r   r   keysr?   lowerr   valuesrx   ry   listr@   r   r^   indexrA   )	rB   bos_token_idtranslate_token_idtranscribe_token_idnotimestamps_token_idlangslanguage_idis_language_codebos_sequences	            rH   r   z"WhisperTokenizerFast.prefix_tokens  s    112IJ!77H"889IJ $ : :;M Ninn&'==$ MM//1DM}} 00.t}}="2"9"9";;"mm#&t}}#5#:  ,T]]O <;K-4467QUVfVkVkVmQnoopr 
 99 yy( #5dii[@UV^U_!`aa$~==$q 05;;{3K KL99 tyyL7P 3Vhi&& 56rQ   c                     || j                   |z   | j                  gz   S | j                   |z   |z   | j                  gz   S )z=Build model inputs from a sequence by appending eos_token_id.)r   r   )rB   token_ids_0token_ids_1s      rH    build_inputs_with_special_tokensz5WhisperTokenizerFast.build_inputs_with_special_tokens  sK    %%3t7H7H6III!!K/+=ARAR@SSSrQ   r   r   already_has_special_tokensc                     |rt         |   ||d      S dgt        | j                        z  }dg}||dgt        |      z  z   |z   S |dgt        |      z  z   dgt        |      z  z   |z   S )a  
        Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding
        special tokens using the tokenizer `prepare_for_model` method.

        Args:
            token_ids_0 (`List[int]`):
                List of IDs.
            token_ids_1 (`List[int]`, *optional*):
                Optional second list of IDs for sequence pairs.
            already_has_special_tokens (`bool`, *optional*, defaults to `False`):
                Whether or not the token list is already formatted with special tokens for the model.

        Returns:
            `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
        T)r   r   r   r   r   )r/   get_special_tokens_maskrx   r   )rB   r   r   r   prefix_onessuffix_onesrG   s         rH   r   z,WhisperTokenizerFast.get_special_tokens_mask  s    & &72'[]a 3   cC 2 233c1#K(8"89KGGqcC$445!s;?O9OPS^^^rQ   c                     | j                  |||        | j                  dd  }t        |      D cg c]  \  }}|dz   |f }}}|S c c}}w )N)r@   r?   rA   r   )r   r   r\   )rB   r@   r?   no_timestampsforced_tokensrankrj   forced_decoder_idss           rH   get_decoder_prompt_idsz+WhisperTokenizerFast.get_decoder_prompt_ids>  sa    D8TaPab
 **12.CL]C[\KD%tax/\\!! ]s   A	c                "    t        | ||||      S )N)return_timestampsreturn_languagerb   )r   )rB   model_outputsr   r   rb   s        rH   r   z WhisperTokenizerFast._decode_asrH  s    /+)
 	
rQ   rr   c                        dd|j                         z   d      }|d   dd }t         fd|D        d      }|  j                  |      }t        d	| d
      |j	                  |       |d   S )z`Converts prompt text to IDs that can be passed to [`~WhisperForConditionalGeneration.generate`].r   r   F)add_special_tokensr   r   Nc              3   H   K   | ]  }|j                   d    k\  s|  yw)r   N)r[   ).0xrB   s     rH   	<genexpr>z6WhisperTokenizerFast.get_prompt_ids.<locals>.<genexpr>X  s$      \qqDDXDXYZD[?[ \s   ""zJEncountered text in the prompt corresponding to disallowed special token: r   )tensor_type)stripnextr   ry   convert_to_tensors)rB   rr   return_tensorsbatch_encodingprompt_text_idsspecial_token_idrj   s   `      rH   get_prompt_idsz#WhisperTokenizerFast.get_prompt_idsR  s    /tzz|1CX]^ )5ab9 \O \^bc'../?@Eijoippqrss))n)Ek**rQ   ra   r   r   c                     t        |t              s| j                  |      }|s|S |d   |k(  }|r||v r||j                  |      d  S g S |S )Nr   )r-   r   _convert_to_listr   )rB   ra   r   r   
has_prompts        rH   r   z"WhisperTokenizerFast._strip_prompta  sc    )T*--i8I q\_4
%2 1G!H!JKK	rQ   c                 v   t        | d      r\dt        t        |             v r| j                         j	                         } nMdt        t        |             v r7| j	                         } n&dt        t        |             v r| j                         } t        | t        j                        r| j                         } | S )Nnumpyrp   
tensorflowjaxlib)	rs   r.   r(   rq   r  tolistr-   ru   ndarray)ra   s    rH   r  z%WhisperTokenizerFast._convert_to_lists  s     9g&#d9o..%MMO113	T)_!55%OO-	T)_--!((*Ii,!((*IrQ   )NNNN<|endoftext|>r  r  FNNF)F{Gz?  )r  r  )r  )F)FNFr  FFFF)N)NNN)NF)NNT)ru   )-rM   
__module____qualname____doc__r   vocab_files_namesmodel_input_namesr   slow_tokenizer_classr0   r   rN   rS   r.   rn   r   r   r   boolr}   r   r]   r_   r~   r   r   r   staticmethodr   r   r   r   r   propertyr   intr   r   r   r   r   r  r   r  __classcell__)rG   s   @rH   r   r   ,   s   $L *$&67+ !!! =5@;] ;5} 5 W[( 	( VBH j jD &9 %*-1$ $', %"'L "L '+	L
 L L !%L L L  L 
L^ (-egl $?C`d	$O     1c 1HSM 1]bcf]g 1$
# $
C $
\` $
L tCy  DTQUVYQZ T sx_9_3;DI3F_ko_	c_>"
+3 +tCy 3 `c $  rQ   r   )%r  r2   r   r<   r   	functoolsr   typingr   r   r   r  ru   
tokenizersr   r   r	   tokenization_utils_baser   tokenization_utils_fastr   utilsr   english_normalizerr   r   tokenization_whisperr   r   r   r   r   
get_loggerrM   loggerr   r   r,   rQ   rH   <module>r#     ss    (  	 	   ( (  = = 4 >  J f f 
		H	% &(	 U	2 U	rQ   