
    sgP"                         d Z ddlmZ ddlmZmZmZmZ ddlm	Z	m
Z
mZ ddlmZ ddlmZmZ ddlmZ  ej&                  e      Z G d	 d
e      Z G d de      Zy)zGPT-J model configuration    )OrderedDict)AnyListMappingOptional   )PreTrainedTokenizer
TensorTypeis_torch_available)PretrainedConfig)OnnxConfigWithPastPatchingSpec)loggingc                   X     e Zd ZdZdZdddddZ	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d	 fd	Z xZS )

GPTJConfiga=  
    This is the configuration class to store the configuration of a [`GPTJModel`]. It is used to instantiate a GPT-J
    model according to the specified arguments, defining the model architecture. Instantiating a configuration with the
    defaults will yield a similar configuration to that of the GPT-J
    [EleutherAI/gpt-j-6B](https://huggingface.co/EleutherAI/gpt-j-6B) architecture. Configuration objects inherit from
    [`PretrainedConfig`] and can be used to control the model outputs. Read the documentation from [`PretrainedConfig`]
    for more information.

    Args:
        vocab_size (`int`, *optional*, defaults to 50400):
            Vocabulary size of the GPT-J model. Defines the number of different tokens that can be represented by the
            `inputs_ids` passed when calling [`GPTJModel`].
        n_positions (`int`, *optional*, defaults to 2048):
            The maximum sequence length that this model might ever be used with. Typically set this to something large
            just in case (e.g., 512 or 1024 or 2048).
        n_embd (`int`, *optional*, defaults to 4096):
            Dimensionality of the embeddings and hidden states.
        n_layer (`int`, *optional*, defaults to 28):
            Number of hidden layers in the Transformer encoder.
        n_head (`int`, *optional*, defaults to 16):
            Number of attention heads for each attention layer in the Transformer encoder.
        rotary_dim (`int`, *optional*, defaults to 64):
            Number of dimensions in the embedding that Rotary Position Embedding is applied to.
        n_inner (`int`, *optional*, defaults to None):
            Dimensionality of the inner feed-forward layers. `None` will set it to 4 times n_embd
        activation_function (`str`, *optional*, defaults to `"gelu_new"`):
            Activation function, to be selected in the list `["relu", "silu", "gelu", "tanh", "gelu_new"]`.
        resid_pdrop (`float`, *optional*, defaults to 0.1):
            The dropout probability for all fully connected layers in the embeddings, encoder, and pooler.
        embd_pdrop (`int`, *optional*, defaults to 0.1):
            The dropout ratio for the embeddings.
        attn_pdrop (`float`, *optional*, defaults to 0.1):
            The dropout ratio for the attention.
        layer_norm_epsilon (`float`, *optional*, defaults to 1e-5):
            The epsilon to use in the layer normalization layers.
        initializer_range (`float`, *optional*, defaults to 0.02):
            The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
        use_cache (`bool`, *optional*, defaults to `True`):
            Whether or not the model should return the last key/values attentions (not used by all models).

    Example:

    ```python
    >>> from transformers import GPTJModel, GPTJConfig

    >>> # Initializing a GPT-J 6B configuration
    >>> configuration = GPTJConfig()

    >>> # Initializing a model from the configuration
    >>> model = GPTJModel(configuration)

    >>> # Accessing the model configuration
    >>> configuration = model.config
    ```gptjn_positionsn_embdn_headn_layer)max_position_embeddingshidden_sizenum_attention_headsnum_hidden_layersc                    || _         || _        || _        || _        || _        || _        || _        || _        |	| _        |
| _	        || _
        || _        || _        || _        || _        || _        t!        | D  d|||d| y )N)bos_token_ideos_token_idtie_word_embeddings )
vocab_sizer   r   r   r   n_inner
rotary_dimactivation_functionresid_pdrop
embd_pdrop
attn_pdroplayer_norm_epsiloninitializer_range	use_cacher   r   super__init__)selfr    r   r   r   r   r"   r!   r#   r$   r%   r&   r'   r(   r)   r   r   r   kwargs	__class__s                      ^/var/www/html/venv/lib/python3.12/site-packages/transformers/models/gptj/configuration_gptj.pyr+   zGPTJConfig.__init__]   s    * %&$#6 &$$"4!2"(( 	
%LVi	
ms	
    )i  i   i         @   Ngelu_new        r5   r5   gh㈵>g{Gz?TP  r6   F)__name__
__module____qualname____doc__
model_typeattribute_mapr+   __classcell__r.   s   @r/   r   r      sb    5n J#0'&	M &!%)
 )
r0   r   c                        e Zd Z	 	 	 ddededee   def fdZe	de
ee
eef   f   fd       Ze	defd       Ze	defd	       Z	 	 	 	 dd
ededededee   de
eef   f fdZe	defd       Z xZS )GPTJOnnxConfigconfigtaskpatching_specsuse_pastc                 ~    t         |   ||||       t        | j                  dd       sd| j                  _        y y )N)rB   rC   rD   pad_token_idr   )r*   r+   getattr_configrF   )r,   rA   rB   rC   rD   r.   s        r/   r+   zGPTJOnnxConfig.__init__   s=     	d>T\]t||^T:()DLL% ;r0   returnc                     t        ddddi      }| j                  r| j                  |d       ddd|d<   |S ddd|d<   |S )	N	input_idsbatchsequence)r      inputs)	directionzpast_sequence + sequenceattention_mask)r   rD   fill_with_past_key_values_)r,   common_inputss     r/   rO   zGPTJOnnxConfig.inputs   sa    #[g*2M$NO==++MX+N29>X.YM*+  3:j.IM*+r0   c                 .    | j                   j                  S N)rH   r   r,   s    r/   
num_layerszGPTJOnnxConfig.num_layers   s    ||###r0   c                 .    | j                   j                  S rU   )rH   r   rV   s    r/   r   z"GPTJOnnxConfig.num_attention_heads   s    ||"""r0   	tokenizer
batch_size
seq_lengthis_pair	frameworkc                 h   t         t        |   |||||      }t        d|d   i      }| j                  rt               st        d      dd l}|d   j                  \  }	}
|
dz   }|	| j                  || j                  j                  | j                  z  f}t        | j                        D cg c]$  }|j                  |      |j                  |      f& c}|d<   |d   |d<   | j                  r<|d   j                  }j!                  |d   |j#                  	|      gd	
      |d<   |S c c}w )N)rZ   r[   r\   r]   rK   zACannot generate dummy past_keys inputs without PyTorch installed.r      past_key_valuesrQ   )dtyperN   )dim)r*   r   generate_dummy_inputsr   rD   r   
ValueErrortorchshaper   rH   r   rangerW   zerosra   catones)r,   rY   rZ   r[   r\   r]   rS   ordered_inputsre   rL   seqlenpast_key_values_length
past_shape_
mask_dtyper.   s                  r/   rc   z$GPTJOnnxConfig.generate_dummy_inputs   s^    0$M*W`i N 

 %k=3M%NO ==%' !dee -k : @ @v)/!&,,*LL,,0H0HH	
 QVVZVeVePf5KLU[[,ekk*.EF501 ,99I+J'(=='(89??J/4yy 015::eE[cm:3nouv 09 0N+, 5s   .)D/c                      y)N   r   rV   s    r/   default_onnx_opsetz!GPTJOnnxConfig.default_onnx_opset   s    r0   )defaultNF)ru   FN)r7   r8   r9   r   strr   r   boolr+   propertyr   intrO   rW   r   r	   r   r
   r   rc   rs   r=   r>   s   @r/   r@   r@      s    -1
* 
* 
* \*	
*
 
* WS#X%6 67   $C $ $ #S # # *.*&* * 	*
 * J'* 
c	*X C  r0   r@   N)r:   collectionsr   typingr   r   r   r    r	   r
   r   configuration_utilsr   onnxr   r   utilsr   
get_loggerr7   loggerr   r@   r   r0   r/   <module>r      sU      # / / C C 3 4  
		H	%i
! i
ZN' Nr0   