
    sg>                         d Z ddlmZ ddlmZ ddlmZ  ej                  e      Z	 G d de      Z
 G d d	e      Zd	dgZy
)zMoshi model configuration   )PretrainedConfig)logging   )
AutoConfigc                   R     e Zd ZdZdZdgZ	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d fd	Z xZS )MoshiDepthConfiga>  
    This is the configuration class to store the configuration of a [`MoshiDepthDecoder`]. It is used to instantiate a
    Moshi depth decoder model according to the specified arguments, defining the Moshi depth decoder config.

    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
    documentation from [`PretrainedConfig`] for more information.

    Args:
        vocab_size (`int`, *optional*, defaults to 32000):
            Vocabulary size of the MoshiDepthDecoder model. Defines the number of different tokens that can be
            represented by the `inputs_ids` passed when calling [`MoshiDepthDecoder`].
        hidden_size (`int`, *optional*, defaults to 1024):
            Dimensionality of the layers and the pooler layer of the depth decoder.
        input_size (`int`, *optional*, defaults to 4096):
            Dimensionality of the input hidden states. Used to connect the main decoder to the depth decoder.
        num_hidden_layers (`int`, *optional*, defaults to 6):
            Number of depth decoder layers.
        num_attention_heads (`int`, *optional*, defaults to 16):
            Number of attention heads for each attention layer in the depth decoder block.
        num_key_value_heads (`int`, *optional*):
            This is the number of key_value heads that should be used to implement Grouped Query Attention. If
            `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
            `num_key_value_heads=1` the model will use Multi Query Attention (MQA) otherwise GQA is used. When
            converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed
            by meanpooling all the original heads within that group. For more details checkout [this
            paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not specified, will default to `num_attention_heads`.
        audio_vocab_size (`int`, *optional*, defaults to 2048):
            Vocabulary size of the audio part of model. Defines the number of different tokens that can be
            represented by the `audio_codes` passed when calling the Moshi models.
        max_position_embeddings (`int`, *optional*, defaults to 9):
            The maximum sequence length that this model might ever be used with. Typically, set this to something large
            just in case (e.g., 512 or 1024 or 2048).
        hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):
            The non-linear activation function (function or string) in the depth decoder.
        head_dim (`int`, *optional*, defaults to `hidden_size // num_attention_heads`):
            The attention head dimension.
        initializer_range (`float`, *optional*, defaults to 0.02):
            The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
        use_cache (`bool`, *optional*, defaults to `True`):
            Whether or not the model should return the last key/values attentions (not used by all models). Only
            relevant if `config.is_decoder=True`.
        sliding_window (`int`, *optional*, defaults to 8):
            Sliding window attention window size. If not specified, will default to `8`.
        attention_dropout (`float`, *optional*, defaults to 0.0):
            The dropout ratio for the attention probabilities.
        ffn_dim (`int`, *optional*, defaults to 5632):
            Dimensionality of the "intermediate" (often named feed-forward) layer in the depth decoder block. Must be even.
        rms_norm_eps (`float`, *optional*, defaults to 1e-08):
            The epsilon used by the rms normalization layers.
        num_codebooks (`int`, *optional*, defaults to 8):
            The number of audio codebooks for each audio channels.
        tie_word_embeddings (`bool`, *optional*, defaults to `False`):
            Whether to tie weight embeddings
        kwargs (*optional*):
            Dictionary of keyword arguments. Notably:
                - **audio_encoder_config** ([`PretrainedConfig`], *optional*) -- An instance of a configuration object that
                  defines the audio encoder config.

    Example:

    ```python
    >>> from transformers import (
    ...     MoshiDepthConfig,
    ...     MoshiDepthDecoder,
    ... )

    >>> configuration = MoshiDepthConfig()

    >>> # Initializing a MoshiDepthDecoder (with random weights) from the kmhf/hf-moshiko style configuration
    >>> model = MoshiDepthDecoder(configuration)

    >>> # Accessing the model configuration
    >>> configuration = model.config
    ```moshi_depthpast_key_valuesc                 Z   || _         || _        || _        || _        || _        ||n|| _        || _        |	| _        |
xs ||z  | _        || _	        || _
        || _        || _        |dz  dk(  rt        d| d      || _        || _        || _        || _        t%        | L  dd|i| y )Nr      	`ffn_dim=` must be even.tie_word_embeddings )
vocab_sizehidden_size
input_sizenum_hidden_layersnum_attention_headsnum_key_value_headsmax_position_embeddings
hidden_acthead_diminitializer_range	use_cachesliding_windowattention_dropout
ValueErrorffn_dimrms_norm_epsnum_codebooksaudio_vocab_sizesuper__init__)selfr   r   r   r   r   r   r"   r   r   r   r   r   r   r   r   r    r!   r   kwargs	__class__s                       `/var/www/html/venv/lib/python3.12/site-packages/transformers/models/moshi/configuration_moshi.pyr$   zMoshiDepthConfig.__init__h   s    , %&$!2#6 :M:Y#6_r '>$$ FK3F$F!2",!2Q;!y	ABB(* 0K-@KFK    ) }  i            Ni   	   siluN{Gz?T           i   :0yE>r1   F)__name__
__module____qualname____doc__
model_typekeys_to_ignore_at_inferencer$   __classcell__r'   s   @r(   r   r      s^    IV J#4"5   !!'*L *Lr)   r   c                        e Zd ZdZdZdgZdeiZ	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d fd	Ze	d        Z
edefd       Z xZS )	MoshiConfiga  
    This is the configuration class to store the configuration of a [`MoshiModel`]. It is used to instantiate a
    Moshi model according to the specified arguments, defining the audio encoder, Moshi depth decoder and Moshi decoder
    configs. Instantiating a configuration with the defaults will yield a similar configuration to that of the Moshiko model,
    e.g. [kmhf/hf-moshiko](https://huggingface.co/kmhf/hf-moshiko)

    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
    documentation from [`PretrainedConfig`] for more information.

    Args:
        vocab_size (`int`, *optional*, defaults to 32000):
            Vocabulary size of the MoshiDecoder model. Defines the number of different tokens that can be
            represented by the `inputs_ids` passed when calling [`MoshiDecoder`].
        hidden_size (`int`, *optional*, defaults to 4096):
            Dimensionality of the layers and the pooler layer of the main decoder.
        num_hidden_layers (`int`, *optional*, defaults to 32):
            Number of decoder layers.
        num_attention_heads (`int`, *optional*, defaults to 32):
            Number of attention heads for each attention layer in the main decoder block.
        num_key_value_heads (`int`, *optional*):
            This is the number of key_value heads that should be used to implement Grouped Query Attention. If
            `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
            `num_key_value_heads=1` the model will use Multi Query Attention (MQA) otherwise GQA is used. When
            converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed
            by meanpooling all the original heads within that group. For more details checkout [this
            paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not specified, will default to `num_attention_heads`.
        audio_vocab_size (`int`, *optional*):
            Vocabulary size of the audio part of model. Defines the number of different tokens that can be
            represented by the `audio_codes` passed when calling the Moshi models.
        max_position_embeddings (`int`, *optional*, defaults to 3000):
            The maximum sequence length that this model might ever be used with. Typically, set this to something large
            just in case (e.g., 512 or 1024 or 2048).
        rope_theta (`float`, *optional*, defaults to 10000.0):
            The base period of the RoPE embeddings.
        hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):
            The non-linear activation function (function or string) in the decoder.
        head_dim (`int`, *optional*, defaults to `hidden_size // num_attention_heads`):
            The attention head dimension.
        initializer_range (`float`, *optional*, defaults to 0.02):
            The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
        use_cache (`bool`, *optional*, defaults to `True`):
            Whether or not the model should return the last key/values attentions (not used by all models). Only
            relevant if `config.is_decoder=True`.
        sliding_window (`int`, *optional*, defaults to 3000):
            Sliding window attention window size. If not specified, will default to `3000`.
        attention_dropout (`float`, *optional*, defaults to 0.0):
            The dropout ratio for the attention probabilities.
        ffn_dim (`int`, *optional*, defaults to 22528):
            Dimensionality of the "intermediate" (often named feed-forward) layer in the main decoder block. Must be even.
        rms_norm_eps (`float`, *optional*, defaults to 1e-08):
            The epsilon used by the rms normalization layers.
        num_codebooks (`int`, *optional*, defaults to 8):
            The number of audio codebooks for each audio channels.
        tie_word_embeddings (`bool`, *optional*, defaults to `False`):
            Whether to tie weight embeddings
        kwargs (*optional*):
            Dictionary of keyword arguments. Notably:
                - **audio_encoder_config** ([`PretrainedConfig`], *optional*) -- An instance of a configuration object that
                  defines the audio encoder config.
                - **depth__config** ([`PretrainedConfig`], *optional*) -- An instance of a configuration object that
                  defines the depth decoder config.


    Example:

    ```python
    >>> from transformers import (
    ...     MoshiConfig,
    ...     MoshiForConditionalGeneration,
    ... )

    >>> configuration = MoshiConfig()

    >>> # Initializing a MoshiForConditionalGeneration (with random weights) from the kmhf/hf-moshiko style configuration
    >>> model = MoshiForConditionalGeneration(configuration)

    >>> # Accessing the model configuration
    >>> configuration = model.config

    >>> # Saving the model, including its configuration
    >>> model.save_pretrained("kmhf/hf-moshiko")

    >>> # loading model and config from pretrained folder
    >>> moshi_config = MoshiConfig.from_pretrained("kmhf/hf-moshiko")
    >>> model = MoshiForConditionalGeneration.from_pretrained("kmhf/hf-moshiko", config=moshi_config)
    ```moshir
   audio_encoder_configc                    || _         || _        || _        || _        ||n|| _        || _        || _        |	| _        |
xs ||z  | _        || _	        || _
        || _        || _        |dz  dk(  rt        d| d      || _        || _        || _        |j#                  di       }|j#                  dd      }t%        j&                  |fi || _        | j                   | j(                  j                   kD  r&t        d| d	| j(                  j                    d
      || j(                  j*                  n|| _        |j#                  di       }|j/                  | j,                  |||d       t1        di || _        t5        | l  dd|i| y )Nr   r   r   r   r?   r8   mimiz`num_codebooks=zX` is greater than the maximum number of codebooks that the audio encoder can deal with (z). Please lower it.depth_decoder_config)r"   r   r   r!   r   r   )r   r   r   r   r   r   
rope_thetar   r   r   r   r   r   r   r   r    r!   popr   	for_modelr?   codebook_sizer"   updater   rB   r#   r$   )r%   r   r   r   r   r   r"   r   rC   r   r   r   r   r   r   r   r    r!   r   r&   r?   audio_encoder_model_typerB   r'   s                          r(   r$   zMoshiConfig.__init__   s   , %&!2#6 :M:Y#6_r '>$$$ FK3F$F!2",!2Q;!y	ABB(*%zz*@"E#7#;#;L&#Q $.$8$89Q$jUi$j! 9 9 G GG!-  1I  JN  Jc  Jc  Jq  Jq  Ir  rE  F 
 8H7OD%%33Ue 	  &zz*@"E##$($9$9)(!.		
 %5$L7K$L!K-@KFKr)   c                 .    | j                   j                  S )N)r?   sampling_rate)r%   s    r(   rJ   zMoshiConfig.sampling_rate6  s    ((666r)   c                 2     | dd|j                         i|S )z
        Instantiate a [`MoshiConfig`] (or a derived class) from an audio encoder configuration.

        Returns:
            [`MoshiConfig`]: An instance of a configuration object
        r?   r   )to_dict)clsr?   r&   s      r(   from_audio_encoder_configz%MoshiConfig.from_audio_encoder_config:  s*      
!5!=!=!?

 	
r)   )r*   r+       rO   NN  g     @r/   Nr0   TrP   r2   i X  r3   r1   F)r4   r5   r6   r7   r8   r9   r   sub_configsr$   propertyrJ   classmethodr   rN   r:   r;   s   @r(   r=   r=      s    Un J#4"5):6K   $!'CLJ 7 7 
.
 
r)   r=   N)r7   configuration_utilsr   utilsr   auto.configuration_autor   
get_loggerr4   loggerr   r=   __all__r   r)   r(   <module>rZ      sV      3  0 
		H	%yL' yLxu
" u
p ,
-r)   