
    sg                     "   d Z ddlmZ ddlZddlmZmZmZmZm	Z	 ddl
mZmZ ddlmZ ddlmZmZmZmZ dd	lmZ dd
lmZ  ej0                  e      Zddddddddddddddddddddddddddddddddddddddddddi ddddddddddd d!d"dd#d$d%dd&d'd(d)d*dd+dd,dd-dd.dd/dd0diddd1dd1ddd2dddd3d4d5d6d7d8ddd9d:dd;d<d=d>d7d8dd9d:d?d@dA	d=d>d7d8dBdCd9d:d?d@dD
i ddEdFdGdHdIdJdKdLdMdNdOdPdQdRdSdTdUdVdWdXdYdZd[d\d]d^d_d`dadbdcdddedfdgdhdidjdkdldmdndodpddqdEdfdgdhdidjdkdldmdndodpdrdddddddddddddddsd6dtd;dudvdwdxdydzd{d|dddzd{ddddddddd}d~dddddddddddddddddddddddd}dZdddddddddddddd
ddddddddddd
ddddddddddd
dddddddddddddddddddddddd
dddddddddddddddd
dddddddddddddddddddddddddddd	ddddddddddddddddddddddddd	ddddddddddd
dZddddddddddd
dddddddÜdĜZdń Z G dƄ dǫ      Z G dȄ de      Z  G dʄ de      Z! G d̄ de      Z" G d΄ de      Z# G dЄ de      Z$e e!e!e"e#e#e#e#e#e$e#e#dҜZ%defdԄZ&y)z
Integration with GGML / The file is copied and adapted from https://github.com/99991/pygguf
with extra methods beings exposed
    )arrayN)	Tokenizerdecodersnormalizerspre_tokenizers
processors)BPEUnigram   )
AddedToken)GPT2ConverterLlamaConverterQwen2ConverterT5Converter)logging)tqdmzmodel.embed_tokenszmodel.layerszmlp.up_projzmlp.down_projzmlp.gate_projpost_attention_layernorminput_layernormzself_attn.q_projzself_attn.v_projzself_attn.k_projzself_attn.o_projzlm_head.weightz
model.norm)
token_embdblkffn_upffn_downffn_gateffn_norm	attn_normattn_qattn_vattn_kattn_outputoutput.weightoutput_normr   r   ffn_up_expszmlp.expertsffn_up_shexpzmlp.shared_expert.up_projffn_down_expsffn_down_shexpzmlp.shared_expert.down_projr   zffn_gate_inp.weightzmlp.gate.weightffn_gate_expsffn_gate_shexpzmlp.shared_expert.gate_projffn_gate_inp_shexpzmlp.shared_expert_gater   r   r   r   r   r    r!   zmlp.gate_up_projzself_attn.qkv_proj)r   r   r   r   r   r   r   attn_qkvr   r    r!   z"transformer.word_embeddings.weightz%transformer.word_embeddings_layernormztransformer.hzmlp.dense_h_to_4hzmlp.dense_4h_to_hzself_attention.query_key_valuezself_attention.denseztransformer.ln_f)ztoken_embd.weighttoken_embd_normr   r   r   r   r   r)   r   r    r!   word_embeddingshz	.lm_head.ln_f)	r   r   r   r   r   r)   r   .output.r!   z.ln_mlp.ln_attn)
r   r   r   r   z.attn_norm.attn_norm_2r)   r   r.   r!   sharedzdec.blk.{bid}.attn_qz+decoder.block.{bid}.layer.0.SelfAttention.qzdec.blk.{bid}.attn_kz+decoder.block.{bid}.layer.0.SelfAttention.kzdec.blk.{bid}.attn_vz+decoder.block.{bid}.layer.0.SelfAttention.vzdec.blk.{bid}.attn_oz+decoder.block.{bid}.layer.0.SelfAttention.ozdec.blk.{bid}.attn_rel_bzAdecoder.block.{bid}.layer.0.SelfAttention.relative_attention_biaszdec.blk.{bid}.attn_normz&decoder.block.{bid}.layer.0.layer_normzdec.blk.{bid}.cross_attn_qz-decoder.block.{bid}.layer.1.EncDecAttention.qzdec.blk.{bid}.cross_attn_kz-decoder.block.{bid}.layer.1.EncDecAttention.kzdec.blk.{bid}.cross_attn_vz-decoder.block.{bid}.layer.1.EncDecAttention.vzdec.blk.{bid}.cross_attn_oz-decoder.block.{bid}.layer.1.EncDecAttention.ozdec.blk.{bid}.cross_attn_normz&decoder.block.{bid}.layer.1.layer_normzdec.blk.{bid}.ffn_gatez/decoder.block.{bid}.layer.2.DenseReluDense.wi_0zdec.blk.{bid}.ffn_upz/decoder.block.{bid}.layer.2.DenseReluDense.wi_1zdec.blk.{bid}.ffn_downz-decoder.block.{bid}.layer.2.DenseReluDense.wozdec.blk.{bid}.ffn_normz&decoder.block.{bid}.layer.2.layer_normzdec.output_normzdecoder.final_layer_normz+encoder.block.{bid}.layer.0.SelfAttention.qz+encoder.block.{bid}.layer.0.SelfAttention.kz+encoder.block.{bid}.layer.0.SelfAttention.vz+encoder.block.{bid}.layer.0.SelfAttention.ozAencoder.block.{bid}.layer.0.SelfAttention.relative_attention_biasz&encoder.block.{bid}.layer.0.layer_normz/encoder.block.{bid}.layer.1.DenseReluDense.wi_0z/encoder.block.{bid}.layer.1.DenseReluDense.wi_1z-encoder.block.{bid}.layer.1.DenseReluDense.woz&encoder.block.{bid}.layer.1.layer_normzencoder.final_layer_norm)enc.blk.{bid}.attn_qenc.blk.{bid}.attn_kenc.blk.{bid}.attn_venc.blk.{bid}.attn_oenc.blk.{bid}.attn_rel_benc.blk.{bid}.attn_normenc.blk.{bid}.ffn_gateenc.blk.{bid}.ffn_upenc.blk.{bid}.ffn_downenc.blk.{bid}.ffn_normenc.output_normr    )r   r2   r3   r4   r5   r6   r7   r8   r9   r:   r;   r<   ztransformer.wteztransformer.wpeln_1zattn.c_attnzattn.c_proj.weightzattn.c_proj.biasln_2zmlp.c_fcz
mlp.c_proj)r   r   position_embdr!   r   r)   zattn_output.weightzattn_output.biasr   r   r   )r   r   r   r   r   r   r   r   r   r   r    r!   zbackbone.embeddingszbackbone.layerszmixer.A_logzmixer.conv1dzmixer.in_projzmixer.out_projzmixer.x_projzmixer.dt_projnormzbackbone.norm_f)r   r   ssm_a
ssm_conv1dssm_inssm_outssm_xssm_dtr   r!   r    )llamamistralqwen2qwen2moephi3bloomfalcon7b	falcon40bt5	t5encoderstablelmgpt2
starcoder2mambanemotron
model_type_model_name_or_path)architecturenamemax_position_embeddingsnum_hidden_layersintermediate_sizehidden_sizehead_dim
rope_thetanum_attention_headsnum_key_value_headsrms_norm_eps
vocab_size)
context_lengthblock_countfeed_forward_lengthembedding_lengthrope.dimension_countrope.freq_baseattention.head_countattention.head_count_kv attention.layer_norm_rms_epsilonrc   num_expertsnum_experts_per_tok)rd   re   rf   rg   rh   ri   rj   rk   rl   rc   expert_countexpert_used_countbos_token_ideos_token_idunk_token_idpad_token_id)ggml.bos_token_idggml.eos_token_idggml.unknown_token_idggml.padding_token_idn_layern_headlayer_norm_epsilon)re   rg   rj   rc   attention.layer_norm_epsilonn_positions
num_layersd_ffd_modeld_kv	num_headsrelative_attention_num_bucketsdecoder_start_token_id)rd   re   rf   rg   zattention.key_lengthrj   rk   r|   z attention.relative_buckets_countr   rc   layer_norm_eps)	rd   re   rf   rg   rh   rj   rk   r|   rc   n_ctxn_embdrf   )re   rd   rg   rf   rj   r|   norm_epsilon)re   rd   rg   rf   rj   rk   r|   conv_kernel
state_sizetime_step_rank)	rc   rd   rg   rl   re   zssm.conv_kernelzssm.state_sizezssm.time_step_rankzssm.inner_sizenorm_eps)generalrG   rH   rI   rJ   falcon	tokenizerrK   rL   rO   rQ   rR   rS   rT   rU   tokenizer_typetokensscores
token_typemergesadd_prefix_space)

ggml.modelzggml.tokenszggml.scoreszggml.token_typezggml.mergesru   rv   rw   rx   zggml.add_space_prefixchat_template)r   r   ru   rv   rw   rx   )r   tokenizer_configc                    t        |t              s|g}t        |      dk(  r|d   }d }n|d   dk7  rt        d      |\  }}|dv rt	        | d         } | S |dv rt        | d         } | S |dv rt        | d         } | S |dv r3t        d	t        |             j                         j                         } | S |d
v rt        | |      } | S )N   r   	   zPReceived multiple types, therefore expected the first type to indicate an array.)r   r   r            
      )      )   )   B)r   )
isinstancelistlen
ValueErrorintfloatboolr   tobytesdecode_gguf_parse_value)_value	data_typearray_data_types      Q/var/www/html/venv/lib/python3.12/site-packages/transformers/integrations/ggml.pyr   r     s    i&K	
9~aL	Q<1opp%."	?..VAY M 
g	vay! M 
c	fQi
 M	 
c	sDL)113::< M 
c	"6?;M    c                       e Zd Zd Zy)GGUFTokenizerSkeletonc                 p   |j                         D ]  \  }}t        | ||        t        | d      s)t        | d      rt        | d      st        d      | j                  }| j
                  }t        |      D ci c]  \  }}|||    c}}t        j                  d       g }t        j                               D ]k  \  }	}
g }t        dt        |	            D ]*  }|	d | |	|d  }}||v s||v s|j                  |||
f       , t        |fdd	      }|j                  |       m t        |d
 d	      }|D cg c]  }|d   |d   f }}|| _        nt| j                  D 	cg c]  }	t!        |	j#                  d             c}	| _        t        | d      s1t        t        | j                              D cg c]  }d  c}| _        t        | d      sg | _        t        | d      sd | _        t        | d      r| j&                  | j(                  | _        y y y c c}}w c c}w c c}	w c c}w )Nr   r   r   z\tokens and scores need to be passed for a LLaMa tokenizer without merges to be instantiated.z:Merges were not in checkpoint, building merges on the fly.r   c                 $    | d      | d      fS )Nr   r    )xvocabs    r   <lambda>z0GGUFTokenizerSkeleton.__init__.<locals>.<lambda>  s    U1Q4[%!+4N r   T)keyreversec                     | d   S )Nr   r   )vals    r   r   z0GGUFTokenizerSkeleton.__init__.<locals>.<lambda>  s
    CF r   r    added_tokensrs   unknown_token_id)itemssetattrhasattrr   r   r   	enumerateloggerwarningr   ranger   appendsortedextendr   tuplesplitr   rs   r   )selfdict_kvr   r   itr   mergepiece_scorelocalindexpiece_lpiece_rr   _r   s                    @r   __init__zGGUFTokenizerSkeleton.__init__  s    KKM 	 DAqD!Q	  tX&4*'$2I r  [[F[[F.7.?@daQq	\@ENNWXF&*5;;=&9 %"{"1c%j1 FE',Ve}eEFmWG&(W->gw%DEF u*NX\]e$% F(:DIF2893s1vs1v&9F9 DK@DLu5S!12LDK4*-23t{{3C-DEtEt^, "Dt^, $D 4+,1B1B1J $ 5 5D 2K,7 A : MEs   =H#H)+!H. 	H3N)__name__
__module____qualname__r   r   r   r   r   r     s    '6r   r   c                   0    e Zd Zd Zd Zd Zd Zd Zd Zy)GGUFLlamaConverterc                     t        |      | _        | j                  | _        i | _        t	        | j                  dd      dk7  | _        y )Nr   rG   )r   protooriginal_tokenizeradditional_kwargsgetattris_llama_3_tokenizerr   tokenizer_dicts     r   r   zGGUFLlamaConverter.__init__  s>    *>:
"&**!#$+DJJ8H'$RV]$]!r   c                 T    t        t        |j                  |j                              S Nr   zipr   r   r   r   s     r   r   zGGUFLlamaConverter.vocab      Cell344r   c                     |j                   S r   r   r   s     r   r   zGGUFLlamaConverter.merges      ||r   c           
         | j                  | j                        }| j                  | j                        }t        |      D ci c]  \  }\  }}|| }}}}|j                  |j
                  |j                     nd }t        |dd       |j
                  |j                     nd }	t        |dd       |j
                  |j                     nd }
t        t        |||dd            }g }t        | j                  d      s^||j                  t        |dd             |	|j                  t        |	dd             |
|j                  t        |
dd             n}t        j                  t        j                  | j                  j                         dk(        d	   }|D ]6  }|j                  t        | j                  j
                  |   dd             8 t#        |      d	k7  r|j%                  |       t#        | j                  j&                        d	k7  r?|j)                  | j                  j&                  D cg c]  }t        |dd       c}       || j*                  d
<   |	| j*                  d<   |
| j*                  d<   | j,                  r>d | j*                  d<   d| j*                  d<   d| j*                  d<   d| j.                  _        |S c c}}}w c c}w )Nrq   rr   T)	unk_tokenfuse_unkbyte_fallbackr   F
normalizedspecialr   r   r   	eos_token	bos_tokenr   clean_up_tokenization_spaceslegacy)r   r   r   r   rs   r   r   rq   r   r	   r   r   r   npwherer   r   r   add_special_tokensr   
add_tokensr   r   r   r   )r   r   vocab_scoresr   r   word_score	bpe_vocabr   r   r   r   special_tokensspecial_tokens_idxidxadded_tokens                   r   r   zGGUFLlamaConverter.tokenizer  s   zz$**-TZZ(6?6MNN!2NT6T1WN	N8=8J8J8VELL!3!34\`	8?~W[8\8hELL!3!34nr	8?~W[8\8hELL!3!34nr	#"
	 tzz<0$%%juVZ&[\$%%juVZ&[\$%%juVZ&[\ "$"((4::3H3H*IQ*N!OPQ!R) j%%j1B1B31GTYcg&hij ~!#((8tzz&&'1,  ]a]g]g]t]tukKE5Iu /8{+.7{+.7{+$$9=D""#56EID""#AB/4D""8,-2D##*k ON vs   KKc                 0   t        j                         t        j                         t        j                  dd      g}| j                  r|t        j
                  ddd      gz  }|r|t        j                  dd      gz  }t        j                  |      S )N   ▁r   FTr   trim_offsets	use_regexr   contentleft)r   ByteFallbackFuseReplacer   	ByteLevelStripSequencer   replacementr   sequences       r   decoderzGGUFLlamaConverter.decoderN  s    !!#MMOUC(
 $$++UQVbfghhH!<==H  **r   c                     | j                  | j                        }| j                  | j                        }|||_        d}d}t        | j                  d      r| j                  j
                  }| j                  ||      }|||_        | j                  ||      |_        | j                         }|r||_        | j                  r7t        j                  ddd      |_        t        j                  g       |_        |S )Nr  Tr   Fr  )r   r   
normalizerr   r   r   pre_tokenizerr  post_processorr   r   r  r   r  )r   r   r  r  r   r  r   s          r   	convertedzGGUFLlamaConverter.converted\  s    NN4::.	 __TZZ0
!#-I 4**,>?#66GG**;8HI$&3I# LL6FG	,,.'5I$ $$&4&>&>!&Ud'I#
 $/#7#7#;I r   N)	r   r   r   r   r   r   r   r  r!  r   r   r   r   r     s"    ^58t+!r   r   c                   *     e Zd Zd Zdef fdZ xZS )GGUFQwen2Converterc                 2    t        |      | _        i | _        y r   r   r   r   r   s     r   r   zGGUFQwen2Converter.__init__      "7"G!#r   returnc           	      4   t        | j                  j                        D ci c]  \  }}||
 }}}| j                  j                  }t        |   ||      }|j                  t        ddd      t        ddd      t        ddd      g       |S c c}}w )N<|endoftext|>FTr   z<|im_start|>z
<|im_end|>)r   r   r   r   superr!  r  r   r   r   r  r   r   r   	__class__s         r   r!  zGGUFQwen2Converter.converted  s    (1$2I2I2P2P(QRWQqRR((//G%eV4	$$?udK>eTJ<E4H	
  Ss   Br   r   r   r   r   r!  __classcell__r,  s   @r   r#  r#    s    $9  r   r#  c                   6    e Zd Zd Zd Zd Zd Zd ZdefdZ	y)	GGUFPhi3Converterc                 T    t        |      | _        | j                  | _        i | _        y r   )r   r   r   r   r   s     r   r   zGGUFPhi3Converter.__init__  s"    *>:
"&**!#r   c                 T    t        t        |j                  |j                              S r   r   r   s     r   r   zGGUFPhi3Converter.vocab  r   r   c                     |j                   S r   r   r   s     r   r   zGGUFPhi3Converter.merges  r   r   c                    | j                  | j                        }| j                  | j                        }t        |      D ci c]  \  }\  }}|| }}}}t	        t        ||            }|j                  t        ddddd      t        ddd      t        dddd      t        d	ddd      t        d
ddd      t        dddd      t        dddd      t        dddd      t        dddd      t        dddd      t        dddd      t        dddd      g       |j                  |j                  |j                     nd | j                  d<   |j                  |j                  |j                     nd | j                  d<   |j                  |j                  |j                     nd | j                  d<   |j                  |j                  |j                     nd | j                  d<   |S c c}}}w )N</s>TF)rstriplstripr   r   r)  r   z<|assistant|>)r7  r   r   z<|placeholder1|>z<|placeholder2|>z<|placeholder3|>z<|placeholder4|>z
<|system|>z<|end|>z<|placeholder5|>z<|placeholder6|>z<|user|>r   r   r   	pad_token)r   r   r   r   r   r	   r  r   rs   r   r   rr   rq   rt   )	r   r   r  r   r   r  r  r  r   s	            r   r   zGGUFPhi3Converter.tokenizer  s    zz$**-TZZ(6?6MNN!2NT6T1WN	Nc)V45	$$6$uX\]?udK?4ESWX-duVZ[-duVZ[-duVZ[-duVZ[<PTU9TeTR-duVZ[-duVZ[:dudS	
$ 160B0B0NELL++,TX 	{+ 160B0B0NELL++,TX 	{+ 160B0B0NELL++,TX 	{+ 160B0B0NELL++,TX 	{+ E Os   G:c                     t        j                         t        j                         t        j                  |d      g}|r|t        j                  dd      gz  }t        j
                  |      S )Nr   r   r  )r   r  r  r  r  r  r  s       r   r  zGGUFPhi3Converter.decoder  s\    !!#MMO[#.
 !<==H  **r   r'  c                     | j                  | j                        }d}d}t        | j                  d      r| j                  j                  }| j                  ||      |_        |S )Nr  Tr   )r   r   r   r   r   r  )r   r   r  r   s       r   r!  zGGUFPhi3Converter.converted  s\    NN4::.	4**,>?#66GG LL6FG	r   N)
r   r   r   r   r   r   r   r  r   r!  r   r   r   r1  r1    s(    $
5%N	+
9 
r   r1  c                   *     e Zd Zd Zdef fdZ xZS )GGUFGPTConverterc                 2    t        |      | _        i | _        y r   r%  r   s     r   r   zGGUFGPTConverter.__init__  r&  r   r'  c                     t        | j                  j                        D ci c]  \  }}||
 }}}| j                  j                  }t        |   ||      }|S c c}}w r   )r   r   r   r   r*  r!  r+  s         r   r!  zGGUFGPTConverter.converted  s]    (1$2I2I2P2P(QRWQqRR((//G%eV4	 Ss   Ar-  r/  s   @r   r=  r=    s    $9  r   r=  c                   0    e Zd Zd Zd Zd Zd ZdefdZy)GGUFT5Converterc                     dg|d<   t        |      | _        t        | j                  j                        D ci c]  \  }}||
 c}}| _        | j                  | _        i | _        y c c}}w )Nz
dummy textr   )r   r   r   r   token2idr   r   )r   r   r   r   s       r   r   zGGUFT5Converter.__init__  s]    $0>x *>:
*3DJJ4E4E*FG$!QAG"&**!# Hs   A(c                 T    t        t        |j                  |j                              S r   r   r   s     r   r   zGGUFT5Converter.vocab  r   r   c                     t        | j                  dd      rcg }t        | j                  dd      r|t        j                  d      gz  }|t        j                  dd      gz  }t        j
                  |      S y )Nr   Tr   r  )prependr   )patternr  )r   r   r   Prependr  r  )r   r   r  s      r   r  zGGUFT5Converter.normalizer  sr    4**Hd;Ht..0BDI[00?@@,,S%HIIH''11r   c                 Z    t        j                  ddgg dd| j                  d   fg      S )N$Ar6  )rJ  r6  z$Br6  )singlepairr  )r   TemplateProcessingrC  )r   s    r   r   zGGUFT5Converter.post_processor  s5    ,,&>-v./
 	
r   r'  c                    | j                  | j                        }t        t        || j                  j                  d            }| j                  | j                        }|||_        d}d}t        | j                  d      r| j                  j                  }| j                  ||      }|||_	        | j                  ||      |_
        | j                         }|r||_        |S )NF)unk_idr   r  Tr   )r   r   r   r
   rs   r  r   r   r   r  r  r   )r   r  r   r  r  r   r  r   s           r   r!  zGGUFT5Converter.converted
  s    zz$**-zz..#
	 __TZZ0
!#-I 4**,>?#66GG**;8HI$&3I# LL6FG	,,.'5I$r   N)	r   r   r   r   r   r  r   r   r!  r   r   r   rA  rA    s"    $5
9 r   rA  )rG   rI   	qwen2_moerK   rL   r   rQ   rR   rS   rO   rT   rU   r'  c                 ^    | }t        |   |      }|j                         }||j                  fS )a6  
    Utilities to convert a slow tokenizer instance in a fast tokenizer instance.

    Args:
        architecture (`str`): The model architecture derived from gguf file.
        transformer_tokenizer ([`~tokenization_utils_base.PreTrainedTokenizer`]):
            Instance of a slow tokenizer to convert in the backend tokenizer for
            [`~tokenization_utils_base.PreTrainedTokenizerFast`].

    Return:
        A instance of [`~tokenizers.Tokenizer`] to be used as the backend tokenizer of a
        [`~tokenization_utils_base.PreTrainedTokenizerFast`]
    )GGUF_TO_FAST_CONVERTERSr!  r   )rX   r   tokenizer_class_name	converterfast_tokenizers        r   convert_gguf_tokenizerrV  :  s9     ('(<=nMI((*N96666r   )'__doc__r   numpyr   
tokenizersr   r   r   r   r   tokenizers.modelsr	   r
    r   convert_slow_tokenizerr   r   r   r   utilsr   utils.loggingr   
get_loggerr   r   GGUF_TENSOR_MAPPINGGGUF_CONFIG_MAPPINGGGUF_TOKENIZER_MAPPINGr   r   r   r#  r1  r=  rA  rR  rV  r   r   r   <module>rc     s   
   S S *  _ _    
		H	%
 +##.&$$$))#  +##.&$$$))#  +##.&$$$))#*~ 	} 	3	
 	 	7 	. 	0 	 	7 	6 	& 	$ 	$ 	$  	)!" 	)#$ 	|%* +$#&.&())# BB%'.&4-)) (%'&4-
 (%'! 4-h M 	 M 	 M	
 	 M 	#$g 	"#K 	%&U 	%&U 	%&U 	%&U 	()Q 	!"S 	 Q 	!"Q  	!"J!" 	5#$ !N M M M$g#K"S Q"Q"J5);@  M M M M$g#K"S Q"Q"J5 +##.&$$$))#  (*)!2.  + .&$$$))# , $!#!() +#.&$$$))#oe T %%
 4*2) *& 5#8,:" 4*2) *& 5#8,:" 4*2) $& 5#8,:" 4*2) $& 5#8,:"%2 4*2) $& 5#8,:" ,+!/!/	 4*2) $& 5#8,:" !) ("(< (#%% & +#8(<,L":" 4*2) $ 5#8(8"
 !!$4 ((< +3)2 5#8(6 #3),@*(&.-
 4*2) $& 5#8,6"e^ D ''++!/!/!3 )"++!/!/ 00(6 (6Vv vr (H HV	} 	<k <@  # "
   7I 7r   