
     sg,!             	      &   d dl mZ d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	Z	d dl
mZ d dlmZ d dlmZ d dlmZmZmZmZmZmZmZmZmZmZ d dlmZ d dlZd dlZd dl Z erd dl m!Z! d	ejD                  vr5e	jF                  jI                  d
 e% ee&      jN                  dz               d dl(Z( ejR                  d      Z* G d de      Z+ edd      Z, G d d      Z-e-j]                  d       G d de-             Z/e-j]                  dd       G d de-             Z0e-j]                  d       G d de-             Z1e-j]                  d       G d de-             Z2e-j]                  d d!       G d" d#e-             Z3e-j]                  d$       G d% d&e-             Z4e-j]                  d'd(       G d) d*e-             Z5e-j]                  d+       G d, d-e-             Z6e-j]                  d.       G d/ d0e-             Z7e-j]                  d1d2d3       G d4 d5e-             Z8e-j]                  d6d7d8d9       G d: d;e-             Z9e-j]                  d<       G d= d>e-             Z:e-j]                  d?       G d@ dAe-             Z;e-j]                  dB       G dC dDe-             Z<e-j]                  dE       G dF dGe-             Z=e-j]                  dH       G dI dJe-             Z>e-j]                  dK       G dL dMe-             Z?e-j]                  dN       G dO dPe-             Z@e-j]                  dQ       G dR dSe-             ZAe-j]                  dT       G dU dVe-             ZBe-j]                  dW       G dX dYe-             ZCe-j]                  dZ       G d[ d\e-             ZDe-j]                  d]       G d^ d_e-             ZEe-j]                  d`       G da dbe-             ZFe-j]                  dc       G dd dee-             ZGe-j]                  dfdgdh       G di dfe-             ZHe-j]                  dj       G dk djeH             ZIe-j]                  dldm       G dn dleH             ZJe-j]                  do       G dp dqe-             ZKe-j]                  dr       G ds dte-             ZLe-j]                  du       G dv dwe-             ZMe-j]                  dx       G dy dze-             ZNe-j]                  d{d|d}       G d~ de-             ZOe-j]                  d       G d de-             ZPe-j]                  d      e-j]                  d       G d de-                    ZQe-j]                  d       G d de-             ZRe-j]                  d       G d de-             ZSe-j]                  dd       G d deH             ZTe-j]                  d       G d de-             ZUe-j]                  d       G d de-             ZVe-j]                  d       G d de-             ZWe-j]                  d      e-j]                  d      e-j]                  d      e-j]                  d       G d de-                                  ZXe-j]                  d       G d de-             ZYe-j]                  d       G d de-             ZZe-j]                  dd       G d de-             Z[e-j]                  d       G d de-             Z\e-j]                  d       G d de-             Z]e-j]                  d       G d de9             Z^e-j]                  d       G d de^             Z_e-j]                  d      e-j]                  d       G d de-                    Z` G d de(j                        ZbddZcddZdddZeefdk(  r ee        yy)    )annotationsN)IntEnum)Path)sha256)
TYPE_CHECKINGAnyCallableContextManagerIterableIteratorLiteralSequenceTypeVarcast)chain)TensorNO_LOCAL_GGUF   zgguf-pyz
hf-to-ggufc                  $    e Zd ZdZdZdZdZdZdZy)SentencePieceTokenTypesr                  N)	__name__
__module____qualname__NORMALUNKNOWNCONTROLUSER_DEFINEDUNUSEDBYTE     I/var/www/html/venv/lib/python3.12/site-packages/bin/convert_hf_to_gguf.pyr   r   $   s     FGGLFDr&   r   AnyModeltype[Model])boundc                     e Zd ZU i Zded<   ded<   ded<   ded<   ded	<   d
ed<   ded<   ded<   ded<   ded<   ded<   ded<   ded<   ded<   ded<   ded<   ded<   ded<   d ed!<   	 	 	 	 	 dA	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dBd#Zed$        ZdCdDd%Zd& Z	dEd'Z
dFdGd(ZdHdId)ZdJdKd*Zd+ ZdLd,ZdMd-ZdNd.Zd/ Zd0 ZdOd1Zd2 Zd3 ZedPd4       ZedQd5       ZedRd6       ZedSd7       ZdTd8ZdUd9ZdVd:ZdWd;Zd< Z dXd=Z!d> Z"d? Z#dYd@Z$y")ZModelzdict[str, type[Model]]_model_classesr   	dir_modelgguf.LlamaFileTypeftype	fname_outboolis_big_endianzgguf.GGUFEndian	endianessuse_temp_filelazy	list[str]
part_namesis_safetensorszdict[str, Any]hparamsintblock_countzgguf.TensorNameMap
tensor_mapzset[str] | Nonetensor_nameszgguf.GGUFWritergguf_writer
str | None
model_namePath | Nonemetadata_overridedir_model_cardzgguf.MODEL_ARCH
model_archNc           
        t        |       t        u r!t        t        |       j                  d      || _        || _        || _        || _        |rt        j                  j                  nt        j                  j                  | _        || _        | | _        t        j                  | j                  dd      | _        t#        | j                         dkD  | _        | j$                  s&t        j                  | j                  dd      | _        |t        j'                  | j                        n|| _        | j+                  g d      | _        t        j.                  | j0                  | j,                        | _        d | _        || _        || _        || _        | j
                  t        j<                  j>                  k(  rtA        | jC                               \  }}|jD                  tF        jH                  k(  rCtJ        jM                  d|jD                   d	       t        j<                  jN                  | _        nBtJ        jM                  d
|jD                   d	       t        j<                  jP                  | _        t        jR                  d t        jT                  | j0                     | j                  | j                  |	|
||      | _+        y )Nz$ should not be directly instantiatedmodelz.safetensorsr   pytorch_modelz.bin)n_layersnum_hidden_layersn_layer
num_layersz/choosing --outtype f16 from first tensor type ()z0choosing --outtype bf16 from first tensor type ()patharchr4   r5   split_max_tensorssplit_max_sizedry_runsmall_first_shard),typer,   	TypeErrorr   r.   r0   r1   r3   gguf
GGUFEndianBIGLITTLEr4   r5   r6   get_model_part_namesr8   lenr9   load_hparamsr:   find_hparamr<   get_tensor_name_maprE   r=   r>   rC   rA   rD   LlamaFileTypeGUESSEDnextget_tensorsdtypetorchfloat16loggerinfo
MOSTLY_F16MOSTLY_BF16
GGUFWriterMODEL_ARCH_NAMESr?   )selfr.   r0   r1   r3   r5   eagerrC   rA   rP   rQ   rR   rS   r:   _first_tensors                   r'   __init__zModel.__init__H   s:   
 :tDz2255YZ[["
"*0=,,4??CYCY*I	44T^^Wn]!$//2Q6""#88Z`aDO=D_u))$..9RY++,fg224??DDTDTU !2$' ::++333"4#3#3#56OA|!!U]]2MlN`N`Maabcd!//::
N|OaOaNbbcde!//;;
  ??4;P;PQUQ`Q`;amqm{m{  LP  L^  L^=N_mw~  Sder&   c                P    d| j                   vrt        d| j                        y )NrE   z"Missing property 'model_arch' for )__dict__rU   r   )clss    r'   __init_subclass__zModel.__init_subclass__r   s-     s||+@@PQRR ,r&   c                r     t         fd|D        d       }| j                  |   S |ry t        d|       )Nc              3  @   K   | ]  }|j                   v s|  y wN)r:   ).0krl   s     r'   	<genexpr>z$Model.find_hparam.<locals>.<genexpr>z   s     9!qDLL'8A9s   zcould not find any of: )ra   r:   KeyError)rl   keysoptionalkeys   `   r'   r]   zModel.find_hparamy   sB    9t94@?<<$$0788r&   c                $    | j                          y rw   _set_vocab_gpt2rl   s    r'   	set_vocabzModel.set_vocab       r&   c           	   #    K   t               }| j                  rdnd}|dz  }| j                  |z  }|j                         rt               | _        t
        j                  d| d       t        |dd      5 }t        j                  |      }|j                  d	      t        t              st        d
|      | j                  j                  j                                d d d        n	|| _        i | j                   D ][  }t
        j                  d| d       | j                  r4ddlm} t'        t(        t*            || j                  |z  dd            }nBt-        j.                  t1        j                  t3        | j                  |z        ddd            }|5 }	|j                  |	j                                |	j                         D ]  }
| j                  rE| j4                  r'|	j7                  |
      }t8        j;                  |      }n8|	j=                  |
      }n&|	|
   }| j4                  rt8        j?                  |      }|
|f  	 d d d        ^ tA        |jC                  | j                              dkD  rtE        | j                  jG                  |            }tE        |jG                  | j                              }tE        t        fd|D                    }tA        |      dk(  rtA        |      dkD  rt        d|       t        d| d|       y # 1 sw Y   =xY w# 1 sw Y   >xY ww)Nzmodel.safetensorszpytorch_model.binz.index.jsonz%gguf: loading model weight map from ''rutf-8encoding
weight_mapzCan't load 'weight_map' from zgguf: loading model part 'r   )	safe_openptcpu)	frameworkdeviceT)map_locationmmapweights_onlyc              3  2   K   | ]  }|v s|     y wrw   r%   )rx   nr   s     r'   rz   z$Model.get_tensors.<locals>.<genexpr>   s     &Yjz!}&Ys   	
z#Missing or incomplete model files: zOMismatch between weight map and model parts for tensor names:
Missing tensors: z
Extra tensors: )$setr9   r.   is_filer>   rf   rg   openjsonloadget
isinstancedict
ValueErrorupdater|   r8   safetensorsr   r   r
   r   
contextlibnullcontextrd   strr6   	get_sliceLazyTorchTensorfrom_safetensors_slice
get_tensor
from_eagerr[   symmetric_differencesorted
difference)rl   tensor_names_from_parts
index_name
index_filefindex	part_namer   ctx
model_partnamedatamissingextramissing_filesr   s                  @r'   rb   zModel.get_tensors   s    ,/E,0,?,?(EX
m#
^^j0
 #DKK?
|1MNj#8 <A(,		!"YY|4
%Z
D-I$'DZN%STT!!(():;< < !8DJ 	%IKK4YKqAB""1>#.	$..9:T`dmr0st ,,UZZDNNY<V8Wfkrv  FJ  .K  L %
'..z/@A&OO- %D**99#-#7#7#=D#2#I#I$#OD#-#8#8#>D)$/99#2#=#=d#CD*$%% %	%4 &;;D<M<MNORSST..99:QRSG2==d>O>OPQE"3&Yg&Y#YZM5zQ3}#5#9 #F}o!VWW  "55<I >338'"; < < TI< <&% %s:   A5M8A0L-(B?M'B2L:CM-L72M:M	?Mc                    |t         j                  | j                     vrt        d|d| j                        t         j                  |   }d|v r|J |j                  |      }||z   S )NzMissing z for MODEL_TENSORS of {bid}bid)rV   MODEL_TENSORSrE   r   TENSOR_NAMESformat)rl   r~   r   suffixr   s        r'   format_tensor_namezModel.format_tensor_name   sr    d((99xw.DT__DWXYY%%c*d??"?;;3;'Df}r&   c                    |t         j                  | j                     vryt         j                  |   }d|v r|y|j	                  |      }n|y|||z   k(  S )NFr   r   )rV   r   rE   r   r   )rl   r   r~   r   r   key_names         r'   match_model_tensor_namezModel.match_model_tensor_name   sd    d((99))#.h{3/H6)**r&   c                `    | j                   j                  ||      }|t        d|      |S )N)r~   try_suffixeszCan not map tensor )r=   get_namer   )rl   r   r   new_names       r'   map_tensor_namezModel.map_tensor_name   s9    ??++<+P24(;<<r&   c                v   | j                   j                  | j                         | j                  ddgd      x}3| j                   j	                  |       t
        j                  d|        | j                  ddg      }| j                   j                  |       t
        j                  d|        | j                  d	d
gd      x}3| j                   j                  |       t
        j                  d|        | j                  ddg      }| j                   j                  |       t
        j                  d|        | j                  j                  d      x}3| j                   j                  |       t
        j                  d|        | j                  j                  d      x}3| j                   j                  |       t
        j                  d|        | j                  j                  d      x}3| j                   j                  |       t
        j                  d|        | j                  g dd      x}3| j                   j                  |       t
        j                  d|        | j                  j                  d      x}	3| j                   j!                  |	       t
        j                  d|	        | j                  j                  d      x}
3| j                   j#                  |
       t
        j                  d|
        | j                  j                  d      x}6| j                   j%                  |       | j                   j'                  |       | j                   j)                  | j*                         t
        j                  d| j*                          y )Nmax_position_embeddingsn_ctxTr}   zgguf: context length = hidden_sizen_embdzgguf: embedding length = intermediate_sizen_innerzgguf: feed forward length = num_attention_headsn_headzgguf: head count = num_key_value_headszgguf: key-value head count = 
rope_thetazgguf: rope theta = rms_norm_epszgguf: rms norm epsilon = )layer_norm_epslayer_norm_epsilonnorm_epsilonzgguf: layer norm epsilon = num_local_expertszgguf: expert count = num_experts_per_tokzgguf: experts used count = head_dimgguf: file type = )r?   add_block_countr<   r]   add_context_lengthrf   rg   add_embedding_lengthadd_feed_forward_lengthadd_head_countr:   r   add_head_count_kvadd_rope_freq_baseadd_layer_norm_rms_epsadd_layer_norm_epsadd_expert_countadd_expert_used_countadd_key_lengthadd_value_lengthadd_file_typer0   )rl   r   r   n_ffr   	n_head_kvr   	f_rms_eps
f_norm_eps	n_expertsn_experts_usedr   s               r'   set_gguf_parameterszModel.set_gguf_parameters   s3   (()9)9:%%'@'&JUY%ZZEg//6KK1%9:!!=(";<--f5/x89$$&99%EPT$UUDb44T:KK6tf=>!!#8("CD''/)&23))*?@@IM..y9KK7	{CD,,**<88JE//
;KK-j\:;)).99IF33I>KK3I;?@**+cnr*ssJ  A//
;KK5j\BC))*=>>IK--i8KK/	{;<"ll../DEENR22>BKK5n5EFG((44HA++H5--h7&&tzz2(56r&   c                ,    ~| j                  |      |fgS rw   r   rl   
data_torchr   r   s       r'   modify_tensorszModel.modify_tensors  s    %%d+Z899r&   c                     ~~~~yNFr%   rl   r   r   r   n_dimss        r'   tensor_force_quantzModel.tensor_force_quant  s    (Cr&   c                     y)Nr%   r%   r   s    r'   generate_extra_tensorszModel.generate_extra_tensors  s    r&   c                d    t        d  j                  j                  j                         D              t	        d      z   }t         j                          j                               D ]  \  }}|j                  d      r|j                  }|j                  t        j                  t        j                  fvr|j                  t        j                        }d |j                  d      D ]  }|j                         st!        |       n  j#                  ||      D ]  \  }|j%                         j'                         }t	        |j(                        dk(  r|j'                         }t	        |j(                        } j+                  ||      }|dk  sj                  d      rt,        j.                  j0                  }|du r"t3         fd	t,        j4                  j6                  t,        j4                  j8                  t,        j4                  j:                  t,        j4                  j<                  t,        j4                  j>                  t,        j4                  j@                  t,        j4                  jB                  t,        j4                  jD                  t,        j4                  jF                  f	D              sj                  d
      st,        j.                  j0                  }|du rt3         fdt,        j4                  jH                  t,        j4                  jJ                  fD              rZ jL                  t,        jN                  jP                  t,        jN                  jR                  fv rt,        j.                  jT                  }tW        |tX              r jL                  t,        jN                  jZ                  k(  rt,        j.                  j0                  }nm jL                  t,        jN                  j\                  k(  rt,        j.                  jT                  }n* jL                  t,        jN                  j^                  k(  rt,        j.                  j`                  }n jL                  t,        jN                  jb                  k(  rt,        j.                  jd                  }n jL                  t,        jN                  jP                  k(  rt,        j.                  jf                  }nd jL                  t,        jN                  jR                  k(  rt,        j.                  jh                  }n"tk        d jL                  jl                         	 t,        jn                  jq                  ||      }|j                  tx        jz                  k(  r t-        j|                  |j(                  |      n|j(                  }
ddj                  d t        |
      D               d}tt        j                  d| d dz   d| d|jl                   d|         j                  j                  ||         y # t,        jr                  $ r\}	tt        jw                  d|	d       t,        j.                  jT                  }t,        jn                  jq                  ||      }Y d }	~	1d }	~	ww xY w)Nc              3  8   K   | ]  \  }}t        |        y wrw   )r[   )rx   rn   ss      r'   rz   z(Model.prepare_tensors.<locals>.<genexpr>  s     Oda3q6Os   z.weight,)z.attention.masked_biasz.attention.biasz.rotary_emb.inv_freq.r   r   z_norm.weightFc              3  D   K   | ]  }j                  |        y wrw   r   rx   r~   r   r   rl   s     r'   rz   z(Model.prepare_tensors.<locals>.<genexpr>;  s(       44XsCH    .weightc              3  D   K   | ]  }j                  |        y wrw   r  r  s     r'   rz   z(Model.prepare_tensors.<locals>.<genexpr>M  s(      / 003D/r  zUnknown file type: z%s, %szfalling back to F16{z, c              3  2   K   | ]  }t        |        y wrw   )r   )rx   r   s     r'   rz   z(Model.prepare_tensors.<locals>.<genexpr>v  s     *Ka3q6*Ks   }z%-r  , z --> z
, shape = )	raw_dtype)Dmaxr=   mappingvaluesr[   r   r   rb   endswithrc   rd   re   float32tosplit	isdecimalr;   r   squeezenumpyshaper   rV   GGMLQuantizationTypeF32anyMODEL_TENSORFFN_GATE_INPPOS_EMBDTOKEN_TYPES
SSM_CONV1DTIME_MIX_FIRSTTIME_MIX_W1TIME_MIX_W2TIME_MIX_DECAY_W1TIME_MIX_DECAY_W2
TOKEN_EMBDOUTPUTr0   r_   MOSTLY_TQ1_0MOSTLY_TQ2_0F16r   r2   ALL_F32rh   ri   BF16MOSTLY_Q8_0Q8_0TQ1_0TQ2_0r   r   quantsquantize
QuantErrorrf   warningnpuint8quant_shape_from_byte_shapejoinreversedrg   r?   
add_tensor)rl   max_name_lenr   r   	old_dtypepartr   r   
data_qtypeer  	shape_strr   r   s   `           @@r'   prepare_tensorszModel.prepare_tensors  s$   Odoo.E.E.L.L.NOORUV`Raa %d&A&A&CTEUEUEW X c	RD*}}bc"((I u}}'EE']]5==9
 C

3 >>#d)C
 *.)<)<Zs)S QR$*!))+113 tzz?a'%++-DTZZ?C?V?VW[]egjlr?s
 Q;("3"3N"C!%!:!:!>!>J &  !--:: --66 --99 --88 --<< --99 --99 --?? --??
$  $,,Y7!%!:!:!>!>J&3 / ))44))00 / , zz**77**77& 
 &*%>%>%B%B
 j$/zzT%7%7%?%??%)%>%>%B%B
t'9'9'D'DD%)%>%>%B%B
t'9'9'E'EE%)%>%>%C%C
t'9'9'E'EE%)%>%>%C%C
t'9'9'F'FF%)%>%>%D%D
t'9'9'F'FF%)%>%>%D%D
(+>tzz>O)PQQB;;//jAD UYT^T^bdbjbjTj88ZPptpzpz !*K8E?*K!K LBO	 L>3
!nDEQykQVWaWfWfVggqr{q|}~  ++Hdj+QcQR%c	Rl  BNN8Q0EF!%!:!:!>!>J;;//jADBs    Y  Z/AZ**Z/c                j    | j                   j                  t        j                  j                         y rw   )r?   add_typerV   GGUFTypeMODELr   s    r'   set_typezModel.set_type}  s     !!$--"5"56r&   c           
        | j                   j                         \  }}}}t        j                  j	                  | j
                  | j                  | j                  |      | _        | j                  j                  %| j                  j                  | j                  _	        | j                  j                  ,|dkD  r't        j                  ||||      | j                  _        | j                  j                  j                  d      d   }| j                  j                         r|st        j                   | j                  j                  | j                  j"                  | j                  j$                  | j                  j&                  | j                  j                  ||dk  rdnd       }nlt        j                   | j                  j                  | j                  j"                  | j                  j$                  | j                  j&                  d d d      }| j                  | dz  | _        nF| j                  j(                  t        j*                  | j                  j                  |      z  | _        | j-                          t.        j1                  d	       | j                  j3                  | j                          t.        j1                  d
       | j5                          t.        j1                  d       | j7                          t.        j1                  d       | j                   j9                  t        j:                         y )Nr   rn   r   LoRA)
model_typevocab)
size_labeloutput_typerJ  .ggufzSet meta modelzSet model parameterszSet model tokenizerzSet model quantization version)r?   get_total_parameter_countrV   Metadatar   rC   rD   rA   metadatar   r.   rL  r0   	partitionr1   is_dirnaming_conventionbasenamefinetuneversionparentfill_templated_filenamerG  rf   rg   set_gguf_meta_modelr   r   add_quantization_versionGGML_QUANT_VERSION)rl   
vocab_onlytotal_paramsshared_paramsexpert_paramsexpert_countrM  fname_defaults           r'   prepare_metadatazModel.prepare_metadata  s   CGCSCSCmCmCo@m]L**4+A+A4CVCVX\XgXgiuv ==%!%!4!4DMM ==##+q0@'+|]Taco'pDMM$  ::??44S9!< >>  "%)%;%;DMM<N<NPTP]P]PfPfhlhuhuh~h~  AE  AN  AN  AV  AV  X\  Xe  Xe  Xp  Xp  r}  T`  cd  Td  JP  jn  &o%)%;%;DMM<N<NPTP]P]PfPfhlhuhuh~h~  AE  AN  AN  AV  AV  cg  uy  FM  &N "^^u.EEDN "^^22T5Q5QRVR`R`ReRegr5ssDN$%))$*:*:;*+  ")*4511$2I2IJr&   c                4   | j                          | j                  d       | j                  j                  | j                         | j                  j                          | j                  j                  d       | j                  j                          y )NFr]  rN   T)progress)rB  rc  r?   write_header_to_filer1   write_kv_data_to_filewrite_tensors_to_filecloser   s    r'   writezModel.write  su    /--4>>-B..0...= r&   c                6   t        | j                  j                        dk7  rt        d      | j	                  d       | j                  j                  | j                         | j                  j                          | j                  j                          y )Nr   z)Splitting the vocabulary is not supportedTre  rf  )	r[   r?   tensorsr   rc  rh  r1   ri  rk  r   s    r'   write_vocabzModel.write_vocab  sw    t''(A-HII.--4>>-B..0 r&   c                    g }t        j                  |       D ]7  }|j                  |      s|j                  |      s'|j	                  |       9 |j                          |S rw   )oslistdir
startswithr  appendsort)r.   prefixr   r8   filenames        r'   rZ   zModel.get_model_part_names  sZ     "


9- 	,H""6*x/@/@/H!!(+	, 	r&   c                z    t        | dz  dd      5 }t        j                  |      cd d d        S # 1 sw Y   y xY w)Nconfig.jsonr   r   r   )r   r   r   )r.   r   s     r'   r\   zModel.load_hparams  s5    )m+S7C 	 q99Q<	  	  	 s   1:c                      sJ d fd}|S )Nc                4    D ]  }| j                   |<    | S rw   )r-   )modelclsr   rs   namess     r'   funczModel.register.<locals>.func  s'     4+3""4(4Or&   )r|  r(   returnr(   r%   )rs   r}  r~  s   `` r'   registerzModel.register  s    u	 r&   c                \    	 | j                   |   S # t        $ r t        d|d      d w xY w)NzArchitecture z not supported!)r-   r{   NotImplementedError)rs   rO   s     r'   from_model_architecturezModel.from_model_architecture  sA    	Y%%d++ 	Y%dX_&MNTXX	Ys    +c                   t        |t        t        f      r|j                  d      }n3t        |t              r!|j                         j                  d      }n|}|dv }|xs$ |j                  d      xr |j                  d      }|xs$ |j                  d      xr |j                  d      }|xs$ |j                  d      xr |j                  d	      }|S )
Nr   r   )z<pad>z<mask>z<2mass>z[@BOS@]z<|z|>u   <｜u   ｜>z<unused>)r   bytes	bytearraydecode
memoryviewtobytesrs  r  )rl   token
token_textseems_specials       r'   does_token_look_specialzModel.does_token_look_special  s    eeY/0w7Jz*///AJJ # '
 

 &d**?*?*E*c*J]J]^bJc%h**?*?*G*gJL_L_`fLg &h**?*?	*J*gzObObcfOgr&   c                   g }g }ddl m} |j                  | j                        }| j                  j                  dt        |j                              }t        |j                  j                               |k  sJ | j                  |      }|j                  j                         D ci c]  \  }}||
 }	}}|j                         }
t        |      D ](  }||	vr?|j                  d| d       |j                  t        j                   j"                         G|	|   }||
v r|j$                  |   j&                  s| j)                  |      r*|j                  t        j                   j*                         nt|j-                  dj/                  d      d      }|j                  t        j                   j0                         n)|j                  t        j                   j2                         |j                  |       + |||fS c c}}w )	Nr   AutoTokenizer
vocab_size[PAD]   ▁r   r  )transformersr  from_pretrainedr.   r:   r   r[   rK  r  r  get_vocab_base_preitemsget_added_vocabrangert  rV   	TokenTyper#   added_tokens_decoderspecialr  r!   replacer  r"   r   )rl   tokenstoktypesr  	tokenizerr  tokpreencoded_tokid_reverse_vocabadded_vocabir  s                r'   get_vocab_basezModel.get_vocab_base  s    .!11$..A	\\%%lC	4HI
9??))+,z999((3BK//BWBWBYZ.>k3k)ZZ//1z" 	%A%QCqk* 5 56*1-K' 55a8@@DD`D`afDg (>(>? %o.D.DW.Ms S (C(CDOODNN$9$9:e$	%  x''' [s   +Hc                f   d}|j                  |      }t        t        |      j                               j                         }t        j                  d|        t        j                  d|        d }|dk(  rd}|dk(  rd}|dk(  rd	}|d
k(  rd}|dk(  rd}|dk(  rd}|dk(  rd}|dk(  rd}|dk(  rd}|dk(  rd}|dk(  rd}|dk(  rd}|dk(  rd}|dk(  rd}|dk(  rd }|d!k(  rd"}|dk(  rd#}|d$k(  rd%}|d&k(  rd'}|d(k(  rd)}|d*k(  rd+}|d,k(  rd-}|d.k(  rd/}|d0k(  rd1}|d2k(  rd3}|d4k(  rd5}|d6k(  rd7}|d8k(  rd9}|d:k(  rd;}|d<k(  rd=}|d>k(  rd?}|d@k(  rdA}|dBk(  rdC}|dDk(  rdE}|dFk(  rdG}|
t        j                  dH       t        j                  dI       t        j                  dJ       t        j                  dK       t        j                  dL       t        j                  dM       t        j                  dN       t        j                  dO       t        j                  dP       t        j                  dQ|        t        j                  dI       t        j                  dH       t        dR      t        j                  dSt        |              t        j                  d|        |S )TNu  
 

 


 	 		 	
  
   
    
     
🚀 (normal) 😶‍🌫️ (multiple emojis concatenated) ✅ 🦙🦙 3 33 333 3333 33333 333333 3333333 33333333 3.3 3..3 3...3 កាន់តែពិសេសអាច😁 ?我想在apple工作1314151天～ ------======= нещо на Български ''''''```````""""......!!!!!!?????? I've been 'told he's there, 'RE you sure? 'M not sure I'll make it, 'D you like some tea? We'Ve a'lLzchktok: zchkhsh: @0ef9807a4087ebef797fc749390439009c3b9eda9ad1a097abbe738f486c01e5z	llama-bpe@049ecf7629871e3041641907f3de7c733e4dbfdc736f57d882ba0b0845599754zdeepseek-llm@347715f544604f9118bb75ed199f68779f423cabb20db6de6f31b908d04d7821zdeepseek-coder@8aeee3860c56296a157a1fe2fad249ec40aa59b1bb5709f4ade11c4e6fe652edfalcon@0876d13b50744004aa9aeae05e7b0647eac9d801b5ba4668afc01e709c15e19fzbert-bge@8e62295832751ca1e8f92f2226f403dea30dc5165e448b5bfa05af5340c64ec7zbert-bge-large@b6dc8df998e1cfbdc4eac8243701a65afe638679230920b50d6f17d81c098166mpt@35d91631860c815f952d711435f48d356ebac988362536bed955d43bfa436e34	starcoder@3ce83efda5659b07b1ad37ca97ca5797ea4285d9b9ab0dc679e4a720c9da7454zgpt-2@32d85c31273f8019248f2559fed492d929ea28b17e51d81d3bb36fff23ca72b3	stablelm2@6221ad2852e85ce96f791f476e0b390cf9b474c9e3d1362f53a24a06dc8220ffrefact@9c2227e4dd922002fb81bde4fc02b0483ca4f12911410dee2255e4987644e3f8z	command-r@e636dc30a262dcc0d8c323492e32ae2b70728f4df7dfe9737d9f920a282b8aeaqwen2olmo@a8594e3edff7c29c003940395316294b2c623e09894deebbc65f33f1515df79edbrx@c7699093ba4255a91e702aa38a596aa81669f3525dae06c2953267dde580f448z
jina-v1-enz
jina-v2-en@171aeeedd6fb548d418a7461d053f11b6f1f1fc9b387bd66640d28a4b9f5c643z
jina-v2-es@27949a2493fc4a9f53f5b9b029c82689cfbe5d3a1929bb25e043089e28466de6z
jina-v2-de@c136ed14d01c2745d4f60a9596ae66800e2b61fa45643e72436041855ad4089dz	smaug-bpe@c7ea5862a53e4272c035c8238367063e2b270d51faa48c0f09e9d5b54746c360z	poro-chat@7967bfa498ade6b757b064f31e964dddbb80f8f9a4d68d4ba7998fcf281c531azjina-v2-code@b6e8e1518dc4305be2fe39c313ed643381c4da5db34a98f6a04c093f8afbe99bzchatglm-bpe@7fc505bd3104ca1083b150b17d088b59534ede9bde81f0dd2090967d7fe52ceeviking@b53802fb28e26d645c3a310b34bfe07da813026ec7c7716883404d5e0f8b1901jais@7b3e7548e4308f52a76e8229e4e6cc831195d0d1df43aed21ac6c93da05fec5f	codeshell@63b97e4253352e6f357cc59ea5b583e3a680eaeaf2632188c2b952de2588485etekken@855059429035d75a914d1eda9f10a876752e281a054a7a3d421ef0533e5b6249smollm@3c30d3ad1d6b64202cd222813e7736c2db6e1bd6d67197090fc1211fbc612ae7bloom@bc01ce58980e1db43859146dc51b1758b3b88729b217a74792e9f8d43e479d21zgpt3-finnish@4e2b24cc4770243d65a2c9ec19770a72f08cffc161adbb73fcbb6b7dd45a0aaeexaone@fcace8b9cac38ce847670c970cd5892031a753a1ef381abd1d9af00f713da085zphi-2@60824e3c0d9401f89943cbb2fff727f0e2d4c545ba4df2d6e4f09a6db0f5b450	chameleon@1431a23e583c97432bc230bff598d103ddb5a1f89960c8f1d1051aaa944d0b35z
minerva-7b@8b5a93ed704057481f240da0be7e7dca721d7f8f4755263b6807227a2cbeae65zroberta-bpe
zV**************************************************************************************z5** WARNING: The BPE pre-tokenizer was not recognized!z2**          There are 2 possible reasons for this:zN**          - the model has not been added to convert_hf_to_gguf_update.py yetz>**          - the pre-tokenization config has changed upstreamz`**          Check your model files and convert_hf_to_gguf_update.py and update them accordingly.z<** ref:     https://github.com/ggerganov/llama.cpp/pull/6920z**z** chkhsh:  zBBPE pre-tokenizer was not recognized - update get_vocab_base_pre()ztokenizer.ggml.pre: )	encoder   r   	hexdigestrf   debugr5  r  repr)rl   r  chktxtchktokchkhshress         r'   r  zModel.get_vocab_base_pre  s1    d!!&)F**,-779xx()xx()
 WWCWW CWW"CWWCWWCWW"CWWCWWCWWCWWCWWCWWCWWCWWCWWCWWCWWCWWCWWCWWCWWCWW CWWCWWCWWCWWCWWCWWCWWCWW CWWCWWCWWCWWCWWC;NN4 NNstNNRSNNOPNNklNN[\NN}~NNYZNN4 NN\&23NNstNN4 %&jkk+DI;78xx()
r&   c                |   | j                         \  }}}| j                  j                  d       | j                  j                  |       | j                  j	                  |       | j                  j                  |       t        j                  | j                  d      }|j                  | j                         y )Ngpt2Tload_merges)
r  r?   add_tokenizer_modeladd_tokenizer_preadd_token_listadd_token_typesrV   SpecialVocabr.   add_to_gguf)rl   r  r  r  special_vocabs        r'   r   zModel._set_vocab_gpt2  s    #'#6#6#8 &,,V4**62''/((2))$..dK!!$"2"23r&   c           	        | j                   }| j                  }g }g }ddlm} |j	                  |d      }|d   }t        |j                         j                               |k  sJ | j                  |      }g }	i }
|j                  }|j                         D ]  \  }}||
t        j                  |      <   t        |      dk(  r-t        j                  |||      }t        |      dk(  sJ |	j                  d	j!                  t#        t        j                  |                    |j$                  }i |
|j                         D ci c]  \  }}||
 }}}t'        |      D ]  }||vr?|j                  d
| d       |j                  t(        j*                  j,                         F||   |v r>|j                  ||          |j                  t(        j*                  j.                         |j                  ||          |j                  t(        j*                  j0                          | j2                  j5                  d       | j2                  j7                  |       | j2                  j9                  |       | j2                  j;                  |       t)        j<                  |d      }|	|_        t        |j@                        dk(  r>|jC                  d|j$                  d          |jC                  d|j$                  d          |jC                  d|j$                  d          |jE                  | j2                         y c c}}w )Nr   r  Ttrust_remote_coder  r   max_rankr   r  r  r  r  Fr  bos<|endoftext|>eosunk)#r.   r:   r  r  r  r  	get_vocabr  r  mergeable_ranksr  	QwenModeltoken_bytes_to_stringr[   bpert  r9  mapspecial_tokensr  rV   r  r#   r!   r   r?   r  r  r  r  r  mergesspecial_token_ids_set_special_tokenr  rl   r.   r:   r  r  r  r  r  r  r   rK  r  r  rankmergedr  r  r  r  r  r  s                        r'   _set_vocab_qwenzModel._set_vocab_qwen  s   NN	,, .!11)t1T	\*
9&&(//12Z???((3#33*002 	RKE4<@E)11%895zQ]]?ED]IFv;!###MM#((3y'F'F#OPQ	R  ..C[eC[{C[CaCaCcd/?{C{*ddz" 		7A%QCqk* 5 56q![0mA./ 6 67mA./ 5 56		7 	,,V4**62''/((2)))G%}../14,,UI4L4L_4]^,,UI4L4L_4]^((	0H0H0YZ!!$"2"235 es   
M
c                   | j                         \  }}}| j                  j                  d       | j                  j                  d       | j                  j	                  |       | j                  j                  |       | j                  j                  |       t        j                  | j                  t        |            }|j                  | j                         y Nllamadefaultn_vocab)_create_vocab_sentencepiecer?   r  r  r  add_token_scoresr  rV   r  r.   r[   r  )rl   r  r  scoresr  r  s         r'   _set_vocab_sentencepiecezModel._set_vocab_sentencepiece  s    #'#C#C#E ,,W5**95''/))&1((2))$..#f+N!!$"2"23r&   c           
     	   ddl m} | j                  dz  }|j                         st	        d|        |       }|j                  t        |             | j                  j                  d|j                               }t        |      D cg c]  }d| dj                  d       }}d	g|z  }t        j                  g|z  }t        |j                               D ]  }	|j                  |	      }
|
j                  d      }|j                  |	      }t        j                   }|j#                  |	      rt        j$                  }ne|j'                  |	      rt        j(                  }nC|j+                  |	      rt        j                  }n!|j-                  |	      rt        j.                  }|||	<   |||	<   |||	<    | j                  d
z  }|j                         rt1        |dd      5 }t3        j4                  |      }|D ]W  }||   }	|	|k\  rt6        j9                  d|	 d|dz
          ,|j                  d      ||	<   d||	<   t        j:                  ||	<   Y 	 d d d        | j                  dz  }|j                         rGt1        |dd      5 }t3        j4                  |      }|j                  di       }|j=                         D ]  \  }	}t?        |	      }	|d   }||	   t        j                  k7  rG||	   |j                  d      k7  r0t6        j9                  d|	 d||	   jA                  d      d|       |j                  d      s| jC                  |      rt        j(                  ||	<   n4|jE                  djA                  d      d      }t        j:                  ||	<   d||	<   |j                  d      ||	<    	 d d d        |tG        |      kD  r|tG        |      z
  }t6        jI                  d| d| d       t        d|dz         D ]R  }|jK                  tM        d| dd             |jK                  d       |jK                  t        j                         T |||fS c c}w # 1 sw Y   xY w# 1 sw Y   xY w)Nr   SentencePieceProcessortokenizer.modelFile not found: r  r  r  r        added_tokens.jsonr   r   ignore token : id is out of range, max=r        @tokenizer_config.jsonr  contentreplacing token :  -> r  r  r  Padding vocab with  token(s) - [PAD1] through [PAD)'sentencepiecer  r.   r   FileNotFoundErrorLoadFromFiler   r:   r   r  r  r  r   r#   	IdToPieceGetScorer   	IsUnknownr    	IsControlr!   IsUnusedIsByter$   r   r   r   rf   r5  r"   r  r;   r  r  r  r[   r  rt  r  )rl   r  tokenizer_pathr  r  r  r  r  r  token_idpiecetextscoretoktypeadded_tokens_filer   added_tokens_jsonr~   tokenizer_config_filetokenizer_config_jsonr  
token_datar  	pad_counts                           r'   r  z!Model._create_vocab_sentencepiece  sh   8*;;%%'#&6~6F$GHH*,	s>23\\%%lI4H4H4JK
DI*DUVqaS{11':VV'j:56==>Ki2245 	)H''1E<<(D&&x0E-44G""8,199$$X.199##H-188!!(+166#F8$F8!(HX#	)& !NN-@@$$&'w? 
N1$(IIaL!, NC05H:-xj@Z[ehi[iZj'kl '*zz'':F8$'.F8$)@)M)MHX&N
N !%1H H ((*+S7C =q(,		!%'<'@'@AWY['\$,@,F,F,H =(Hj"8}H!+I!6E)-D-K-KK!(+u||G/DD"NN-=hZr&QYJZJaJabiJjImmqrwqz+{|!~~i0D4P4PQV4W-D-L-L* %o.D.DW.Ms S-D-Q-Q*'.F8$',||G'<F8$==$ F#"S[0ILL.yk9XYbXccdef1i!m, @ed1#QK'BCg& 7 > >?@
 vx''C W2
N 
N= =s    Q+(A2Q0D0Q=0Q:=Rc                   t        j                  | j                        }g }g }g }|j                         D ]9  \  }}}|j	                  |       |j	                  |       |j	                  |       ; t        |      |j                  k(  sJ | j                  j                  d       | j                  j                  d       | j                  j                  |       | j                  j                  |       | j                  j                  |       t        j                  | j                  t        |            }|j                  | j                         y r  )rV   LlamaHfVocabr.   
all_tokensrt  r[   r  r?   r  r  r  r  r  r  r  )	rl   rK  r  r  r  r.  r/  r0  r  s	            r'   _set_vocab_llama_hfzModel._set_vocab_llama_hfM  s   !!$..1$)$4$4$6 	% D%MM$MM% OOG$	%
 6{e.....,,W5**95''/))&1((2))$..#f+N!!$"2"23r&   c           	        t        t        j                  d         dz  d| dz  }t        j	                  dt
        j                  j                  |t        j                                d       t        j                  |d      }|dk(  rd	nd
}|j                  t        j                  j                  j                        }|sJ | j                  j                  t!        |j"                  d         j%                  d             |j                  t        j                  j                  j&                        }| j                  j)                  |r't!        |j"                  d         j%                  d      n|       |j                  t        j                  j                  j*                        }|sJ | j                  j-                  |j.                  D cg c]  }t!        |j"                  |          c}d |        |dk(  r|j                  t        j                  j                  j0                        }|sJ | j                  j3                  |j.                  D cg c]"  }|j"                  |   j5                         d   $ c}d |        |j                  t        j                  j                  j6                        }|sJ | j                  j9                  |j.                  D cg c]"  }|j"                  |   j5                         d   $ c}d |        |dk7  r|j                  t        j                  j                  j:                        }|sJ | j                  j=                  |j.                  D cg c]  }t!        |j"                  |          c}       |j                  t        j                  j                  j>                        x}9| j                  jA                  |j"                  d   j5                         d          |j                  t        j                  j                  jB                        x}9| j                  jE                  |j"                  d   j5                         d          |j                  t        j                  j                  jF                        x}9| j                  jI                  |j"                  d   j5                         d          |j                  t        j                  j                  jJ                        x}9| j                  jM                  |j"                  d   j5                         d          |j                  t        j                  j                  jN                        x}9| j                  jQ                  |j"                  d   j5                         d          |j                  t        j                  j                  jR                        x}:| j                  jU                  |j"                  d   j5                         d          y y c c}w c c}w c c}w c c}w )Nr   modelszggml-vocab-rN  zUsing tokenizer from 'r   r   gpt-neoxr  r
  r   	llama-spm)+r   sysrN   rf   r5  rq  relpathgetcwdrV   
GGUFReader	get_fieldKeys	TokenizerrF  r?   r  r  partsr  PREr  LISTr  r   SCORESr  tolist
TOKEN_TYPEr  MERGESadd_token_mergesBOS_IDadd_bos_token_idEOS_IDadd_eos_token_idUNK_IDadd_unk_token_idPAD_IDadd_pad_token_idADD_BOSadd_add_bos_tokenADD_EOSadd_add_eos_token)rl   rA   r  r+  vocab_readerdefault_prefieldr  s           r'   _set_vocab_builtinzModel._set_vocab_builtinc  s   chhqk*X5+j\QV8WW/PRPYPYP[0\/]]^_`~s;)Z7eY&&tyy':':'@'@Au,,U5;;r?-C-J-J7-ST&&tyy':':'>'>?**UZ5R+A+H+H+Q`kl&&tyy':':'?'?@u''

(S1u{{1~)>(ST_U_(`a$ **499+>+>+E+EFEL5--SXS]S].^au{{1~/D/D/Fq/I.^_j`j.kl&&tyy':':'E'EFu((ejj)Y%++a.*?*?*A!*D)YZe[e)fg$ **499+>+>+E+EFEL5--ejj.YuU[[^/D.YZ!++DII,?,?,F,FGGET--ekk"o.D.D.Fq.IJ!++DII,?,?,F,FGGET--ekk"o.D.D.Fq.IJ!++DII,?,?,F,FGGET--ekk"o.D.D.Fq.IJ!++DII,?,?,F,FGGET--ekk"o.D.D.Fq.IJ!++DII,?,?,G,GHHEU..u{{2/E/E/G/JK!++DII,?,?,G,GHHEU..u{{2/E/E/G/JK V5 )T
 /_ *Z
 /Zs   X6'X;*'Y :Y)
FFFNNr   r   FFN)r.   r   r0   r/   r1   r   r3   r2   r5   r2   rm   r2   rC   rB   rA   r@   rP   r;   rQ   r;   rR   r2   rS   r2   r:   zdict[str, Any] | NoneFr|   zIterable[str]r}   r2   r  r   )r  zIterator[tuple[str, Tensor]])Nr  )r~   gguf.MODEL_TENSORr   
int | Noner   r   r  r   r  )
r   r   r~   ra  r   rb  r   r   r  r2   )r  .bias)r   r   r   zSequence[str]r  r   r   r   r   r   r   rb  r  Iterable[tuple[str, Tensor]]
r   r   r   r   r   rb  r   r;   r  z gguf.GGMLQuantizationType | boolr  rg  )r]  r2   )r.   r   rv  r   r   r   r  r7   )r.   r   )r}  r   r  zCallable[[AnyModel], AnyModel])rO   r   r  r)   )r  zstr | bytesr  r2   )r  z tuple[list[str], list[int], str])r  r   r  None)T)rA   z Literal['gpt-neox', 'llama-spm']r  r;   )%r   r   r   r-   __annotations__rp   classmethodrt   r]   r   rb   r   r   r   r   r   r   r   rB  rG  rc  rl  ro  staticmethodrZ   r\   r  r  r  r  r  r   r  r  r  r:  r^  r%   r&   r'   r,   r,   0   s   -/N*/OO
J""!!  ""  jo<AQUV[SW	(e $(e59(e$/(eDN(e %((e >A(e PT(e %)	(e <Q	(eT S S97<r+,7\:
fRP7/Kb!!         Y Y0(HOd444l
4N(`4,+Lr&   r,   GPTNeoXForCausalLMc                  F    e Zd Zej                  j
                  Zd ZddZy)GPTNeoXModelc                   | j                   d   }| j                  j                  | j                   d          | j                  j                  | j                   d          | j                  j	                  |       | j                  j                  | j                   d          | j                  j                  t        | j                   d   | j                   d   | j                   d   z  z               | j                  j                  | j                   d          | j                  j                  | j                   j                  dd             | j                  j                  | j                   d	          y )
NrJ   r   r   r   
rotary_pctr   use_parallel_residualTr   )r:   r?   r   r   r   r   add_rope_dimension_countr;   r   add_parallel_residualr   r   rl   r<   s     r'   r   z GPTNeoXModel.set_gguf_parameters  s%   ll#67++DLL9R,ST--dll=.IJ((500>Q1RS11\*dll=.IT\\ZoMp.pqr	
 	''5J(KL..t||/?/?@WY]/^_++DLL9I,JKr&   c           
        ~| j                   j                  d| j                   j                  d            }| j                   j                  d| j                   j                  d            }g }t        j                  d|      r|j	                  |d||z  |f      }t        j                  |d d dd d d d f   j	                  d|f      |d d d	d d d d f   j	                  d|f      |d d d
d d d d f   j	                  d|f      fd      }t        j                  d       nt        j                  d|      r|j	                  |d||z  f      }t        j                  |d d dd d f   j	                  |f      |d d d	d d f   j	                  |f      |d d d
d d f   j	                  |f      fd      }t        j                  d       |j                  | j                  |      |f       |S )Nr   r   r   n_embedz9gpt_neox\.layers\.\d+\.attention\.query_key_value\.weightr   r   r>  r   r   dim%re-format attention.linear_qkv.weightz7gpt_neox\.layers\.\d+\.attention\.query_key_value\.bias#re-format attention.linear_qkv.bias)r:   r   rematchreshaperd   catrf   rg   rt  r   	rl   r   r   r   r   ry  rn  qkv_weightsqkv_biass	            r'   r   zGPTNeoXModel.modify_tensors  s   !!(DLL,<,<=R,ST,,""=$,,2B2B92MN,.88PRVW %,,faF9JG-TUK1a
+33RMB1a
+33RMB1a
+33RMB
 J KK?@XXPRVW!))61g6G*HIHQ1W%--wj9Q1W%--wj9Q1W%--wj9
 J KK=>,,T2J?@r&   Nrf  )	r   r   r   rV   
MODEL_ARCHGPTNEOXrE   r   r   r%   r&   r'   rq  rq    s    ((JL$r&   rq  BloomForCausalLM
BloomModelc                  F    e Zd Zej                  j
                  Zd ZddZy)r  c                
   | j                   j                  d| j                   j                  d            }| j                   j                  d| j                   j                  d            }| j                  j                  | j                   j                  d|             | j                  j	                  |       | j                  j                  d|z         | j                  j                  | j                   d          | j                  j                  |       | j                  j                  |       | j                  j                  | j                   d          | j                  j                  | j                         y )	Nr   ry  r   r   
seq_lengthr   rK   r   )r:   r   r?   r   r   r   r   r   r   r   r   r0   )rl   ry  r   s      r'   r   zBloomModel.set_gguf_parameters  s   ,,""=$,,2B2B92MN!!(DLL,<,<=R,ST++DLL,<,<\7,ST--g600W=((i)@A''/**62++DLL9M,NO&&tzz2r&   c           
         ~ j                   j                  d j                   j                  d            } j                   j                  d j                   j                  d            }t        j                  dd|      }g }t        j                  d|      r|j                  |d||z  |f      }t        j                  |d d d	d d d d f   j                  d
|f      |d d dd d d d f   j                  d
|f      |d d dd d d d f   j                  d
|f      fd	      }t        j                  d       nt        j                  d|      r|j                  |d||z  f      }t        j                  |d d d	d d f   j                  |f      |d d dd d f   j                  |f      |d d dd d f   j                  |f      fd	      }t        j                  d       |j                   j                  |      |f       |dk(  r\ j                  J t         fddD              r:|j                   j                  t        j                   j"                        |f       |S )Nr   r   r   ry  ztransformer\. z/h\.\d+\.self_attention\.query_key_value\.weightr   r   r>  r   r   rz  r|  z-h\.\d+\.self_attention\.query_key_value\.biasr}  zword_embeddings.weightc              3  :   K   | ]  }|j                   v  y wrw   r>   rx   r  rl   s     r'   rz   z,BloomModel.modify_tensors.<locals>.<genexpr>       [!1D---[   lm_head.weightzoutput.weight)r:   r   r~  subr  r  rd   r  rf   rg   rt  r   r>   allr   rV   r  r(  r  s	   `        r'   r   zBloomModel.modify_tensors  s5   !!(DLL,<,<=R,ST,,""=$,,2B2B92MNvv&D1,.88FM %,,faF9JG-TUK1a
+33RMB1a
+33RMB1a
+33RMB
 J KK?@XXFM!))61g6G*HIHQ1W%--wj9Q1W%--wj9Q1W%--wj9
 J KK=>,,T2J?@++$$000 [7Z[[ 7 78I8I8P8P QS]^_r&   Nrf  )	r   r   r   rV   r  BLOOMrE   r   r   r%   r&   r'   r  r    s    &&J
3-r&   MPTForCausalLMc                  L    e Zd Zej                  j
                  Zd Zd ZddZ	y)MPTModelc                <   	 | j                          y # t        $ r | j                          | j                  j	                  d       | j                  j                  d       | j                  j                  d       | j                  j                  d       Y y w xY w)NFr   r   r   )r   	Exceptionr  r?   rX  rV  rR  rT  r   s    r'   r   zMPTModel.set_vocab  s~    	1  " 	1))+..u5--a0--a0--a0	1s    BBBc                v   | j                   d   }| j                  j                  | j                   d          | j                  j                  | j                   d          | j                  j	                  |       | j                  j                  d| j                   d   z         | j                  j                  | j                   d          | j                   d   j                  d      x}r| j                  j                  |       | j                  j                  d       | j                   d   d	   +| j                  j                  | j                   d   d	          | j                   d   d
   r,| j                  j                  | j                   d   d          y | j                  j                  d       y )NrI   max_seq_lend_modelr   n_headsattn_config
kv_n_headsh㈵>clip_qkvalibialibi_bias_max        )r:   r?   r   r   r   r   r   r   r   r   add_clamp_kqvadd_max_alibi_bias)rl   r<   r  s      r'   r   zMPTModel.set_gguf_parameters  sb   ll:.++DLL,GH--dll9.EF((500T\\)5L1LM''Y(?@m488FF:F..z:++D1<<&z2>**4<<+Fz+RS<<&w///]0KL\0]^//4r&   c                    ~d|v r&| j                  |d      }|j                  dd      }n| j                  |d      }||fgS )Nscales)r  re  z.scalesr   z
act.scalesrd  )r   r  rl   r   r   r   r   s        r'   r   zMPTModel.modify_tensors*  sY    t++D?^+_H'',?H++D?S+TH:&''r&   Nrf  )
r   r   r   rV   r  MPTrE   r   r   r   r%   r&   r'   r  r  
  s     $$J	15"	(r&   r  OrionForCausalLMc                  D    e Zd Zej                  j
                  Zd Zd Zy)
OrionModelc                $    | j                          y rw   r  r   s    r'   r   zOrionModel.set_vocab:      %%'r&   c                   | j                   d   }| j                   d   }| j                   j                  d|      }d}d| j                   v r| j                   d   }nGd| j                   v r| j                   d   }n)d| j                   v r| j                   d   }nt        d      | j                  j	                  | j
                         | j                  j                  d	       | j                  j                  |       | j                  j                  | j                   d
          | j                  j                  |       | j                  j                  | j                   d          | j                  j                  |       | j                  j                  |       | j                  j                  | j                   d          y )NrJ   r   r   r   max_sequence_lengthr   model_max_length(gguf: can not find ctx length parameter.Meta AI original pthr   r   r   )r:   r   r   r?   r   r0   add_tensor_data_layoutr   r   r   r   r   r   r   rl   r<   
head_counthead_count_kv
ctx_lengths        r'   r   zOrionModel.set_gguf_parameters=  sp   ll#67\\"78
(()>
K
 DLL0&;<J&$,,6&?@J4<</&89JGHH&&tzz2//0FG++J7--dll=.IJ((500>Q1RS''
3**=9 	++DLL,HIr&   N)	r   r   r   rV   r  ORIONrE   r   r   r%   r&   r'   r  r  6  s    &&J(Jr&   r  BaichuanForCausalLMBaiChuanForCausalLMc                  |    e Zd Zej                  j
                  Zd Zd ZddZ	d	d
dZ
	 d		 	 	 	 	 	 	 	 	 ddZddZy)BaichuanModelc                $    | j                          y rw   r  r   s    r'   r   zBaichuanModel.set_vocab]  r  r&   c                Z   | j                   d   }| j                   d   }| j                   j                  d|      }d}d| j                   v r| j                   d   }nGd| j                   v r| j                   d   }n)d| j                   v r| j                   d   }nt        d      | j                  j	                  d	       | j                  j                  |       | j                  j                  | j                   d
          | j                  j                  |       | j                  j                  | j                   d          | j                  j                  | j                   d
   | j                   d   z         | j                  j                  |       | j                  j                  |       | j                  j                  | j                   d          | j                  j                  | j                         | j                   j                  d      d| j                   d   v r| j                   d   j                  d      dk(  r_| j                  j                  t         j"                  j$                         | j                  j'                  | j                   d   d          y y y y NrJ   r   r   r   r  r   r  r  r  r   r   r   rope_scalingfactorrT   linearr:   r   r   r?   r  r   r   r   r   ru  r   r   r   r   r0   add_rope_scaling_typerV   RopeScalingTypeLINEARadd_rope_scaling_factorr  s        r'   r   z!BaichuanModel.set_gguf_parameters`  1   ll#67\\"78
(()>
K
 DLL0&;<J&$,,6&?@J4<</&89JGHH//0FG++J7--dll=.IJ((500>Q1RS11$,,}2MQUQ]Q]^sQt2tu''
3**=9//^0LM&&tzz2<<N+7HUcHd<d||N+//78C  66t7K7K7R7RS  88n9UV^9_` D =e7r&   c                @   | j                   d   }| j                   j                  d|      }g }||d| dk(  rt        j                  d|        | j	                  t
        j                  j                  |      | j                  |d||      f| j	                  t
        j                  j                  |      | j                  |d||      f| j	                  t
        j                  j                  |      | j                  |d      fg}|S | j                  |      |fg}|S )	Nr   r   model.layers.z.self_attn.W_pack.weightzUnpacking and permuting layer r   r   r   )r:   r   rf   rg   r   rV   r  ATTN_Q_reverse_hf_permute_partATTN_KATTN_V_reverse_hf_partr   )rl   r   r   r   r  r  rn  s          r'   r   zBaichuanModel.modify_tensors  s!   \\"78
(()>
K,.?tse;S'TTKK8>?(():):)A)A3G11*aZXZ(():):)A)A3G11*a][](():):)A)A3G))*a8:G  ,,T2J?@Gr&   Nc                    |
||k7  r||z  } |j                   |d|j                  d   |z  dz  g|j                  dd   j                  dd      j                  |j                        S Nr   r   r   r  r  swapaxesrl   weightsr   	n_kv_heads       r'   _reverse_hf_permutez!BaichuanModel._reverse_hf_permute  p     Vy%8y F GOOFAw}}Q'76'AQ'F[WXWYIZ[Xa^WW]]#	
r&   c                l    |j                   d   dz  }| j                  |||z  ||z  |z   df   ||      S Nr   r   .)r  r  )rl   r  n_partr   r   r   s         r'   r  z&BaichuanModel._reverse_hf_permute_part  sI     MM!!''F
1v:>0I30N(OQWYbccr&   c                J    |j                   d   dz  }|||z  ||z  |z   df   S r  )r  )rl   r  r  r   s       r'   r  zBaichuanModel._reverse_hf_part  s6    MM!!q6z!f*q.0#566r&   rf  rw   r  r   r   r;   r  rb  r  r   )
r  r   r  r;   r   r;   r   rb  r  r   )r  r   r  r;   r  r   )r   r   r   rV   r  BAICHUANrE   r   r   r   r  r  r  r%   r&   r'   r  r  Y  se    ))J(a>*
 RVdd'*d47dDNd	d7r&   r  XverseForCausalLMc                  V    e Zd Zej                  j
                  Zd Zd ZddZ	dddZ
y)	XverseModelc                p   | j                   dz  j                         sJ | j                   }| j                  }g }g }ddlm} |j                  |      }|j                  dt        |j                              }t        |j                         j                               }||k\  rt        d      |j                  j                         D 	
ci c]  \  }	}
|
|	
 }}	}
|j                         }t        |      D ]  }||   j!                  d      }|dk(  r0t"        j$                  j&                  }d| d	j!                  d      }nt)        j*                  d
|      rt"        j$                  j&                  }np||   |v rO|j,                  |   j.                  rt"        j$                  j0                  }n5t"        j$                  j2                  }nt"        j$                  j4                  }|j7                  |       |j7                  |        | j8                  j;                  d       | j8                  j=                  d       | j8                  j?                  |       | j8                  jA                  |       t#        jB                  |t        |            }|jE                  | j8                         y c c}
}	w )Ntokenizer.jsonr   r  r  z.Vocabulary size exceeds expected maximum size.r       <r  s   <0x[0-9A-Fa-f]{2}>r	  r
  r  )#r.   r   r:   r  r  r  r   r[   rK  r  r  r  r   r  r  r  r  rV   r  r$   r~  	fullmatchr  r  r!   r"   r   rt  r?   r  r  r  r  r  r  )rl   r.   r:   r  r  r  r  r  max_vocab_indexr  r  r  r  r,  r  r0  r  s                    r'   r   zXverseModel.set_vocab  s1   !11::<<<NN	,,  .!11)<	[[s9??/CD
 i113::<=j(MNNR[RaRaRgRgRi(j>Nk3k)9(j(j//1j) 	%H&x077@JW$..-- A.55g>
4jA..--x(K711(;CC"nn44G"nn99G..//MM*%OOG$#	%& 	,,W5**95''/((2)))S[I!!$"2"239 )ks   J2c                Z   | j                   d   }| j                   d   }| j                   j                  d|      }d}d| j                   v r| j                   d   }nGd| j                   v r| j                   d   }n)d| j                   v r| j                   d   }nt        d      | j                  j	                  d	       | j                  j                  |       | j                  j                  | j                   d
          | j                  j                  |       | j                  j                  | j                   d          | j                  j                  | j                   d
   | j                   d   z         | j                  j                  |       | j                  j                  |       | j                  j                  | j                   d          | j                  j                  | j                         | j                   j                  d      d| j                   d   v r| j                   d   j                  d      dk(  r_| j                  j                  t         j"                  j$                         | j                  j'                  | j                   d   d          y y y y r  r  r  s        r'   r   zXverseModel.set_gguf_parameters  r  r&   c                   ~| j                   d   }| j                   j                  d|      }|j                  d      r| j                  |||      }|j                  d      r| j                  |||      }| j	                  |      |fgS Nr   r   q_proj.weightk_proj.weight)r:   r   r  r  r   )rl   r   r   r   r  r  s         r'   r   zXverseModel.modify_tensors  s    \\"78
(()>
K ==)11*j*UJ==)11*j-XJ%%d+Z899r&   Nc                    |
||k7  r||z  } |j                   |d|j                  d   |z  dz  g|j                  dd   j                  dd      j                  |j                        S r  r  r  s       r'   r  zXverseModel._reverse_hf_permute	  r  r&   rf  rw   r  )r   r   r   rV   r  XVERSErE   r   r   r   r  r%   r&   r'   r  r    s'    ''J-4^a>:
r&   r  FalconForCausalLMRWForCausalLMc                  F    e Zd Zej                  j
                  Zd ZddZy)FalconModelc                t   | j                   j                  d      }|| j                   d   }| j                   j                  d      }|| j                   d   }| j                   j                  d      }|| j                   j                  dd      }| j                  j                  d       | j                  j	                  d	       | j                  j                  | j                   d
          | j                  j                  d| j                   d
   z         | j                  j                  |       | j                  j                  |       | j                  j                  |       | j                  j                  | j                   d          | j                  j                  | j                         y )NrJ   rK   r   r   num_kv_headsr   r      jploskir   r   r   )r:   r   r?   r   r  r   r   r   r   r   r   r   r0   )rl   r<   r   r   s       r'   r   zFalconModel.set_gguf_parameters  s\   ll&&':;,,y1K!!"78>\\(+FLL$$^4	((a8I++D1//	:--dll=.IJ00T\\-5P1PQ((5''/**95++DLL9M,NO&&tzz2r&   c                   ~d|v r| j                  ddg      }| j                  ddgd      xs d}| j                  d	   |z  }|j                  |||z  d
z   |||z        }|d d d df   j                  ||z  ||z        }|d d dgf   j                  ||z  ||z        }	|d d dgf   j                  ||z  ||z        }
t	        j
                  ||	|
f      j                  |      }| j                  |      |fgS )Nquery_key_valuer   r   r  r   Tr   r   r   r   r>  )r]   r:   viewr  rd   r  
reshape_asr   )rl   r   r   r   r   r   r   qkvqry   vs              r'   r   zFalconModel.modify_tensors/  s*    $%%'<h&GHF((.+)FQU(V[Z[I||M2f<H//)Vy-@1-DhPX[aPabCAssF##FX$5x&7HIAAtG$$Y%98f;LMAAtG$$Y%98f;LMAAq!9-88DJ%%d+Z899r&   Nrf  )	r   r   r   rV   r  FALCONrE   r   r   r%   r&   r'   r  r    s    ''J3.:r&   r  GPTBigCodeForCausalLMc                  >    e Zd Zej                  j
                  Zd Zy)StarCoderModelc                n   | j                   d   }| j                  j                  | j                   d          | j                  j                  | j                   d          | j                  j	                  d| j                   d   z         | j                  j                  |       | j                  j                  | j                   d          | j                  j                  d       | j                  j                  | j                   d          | j                  j                  | j                         y )NrK   n_positionsr   r   r   r   r   )r:   r?   r   r   r   r   r   r   r   r   r0   rw  s     r'   r   z"StarCoderModel.set_gguf_parametersN  s    ll9-++DLL,GH--dll8.DE00T\\(5K1KL((5''X(>?**1-++DLL9M,NO&&tzz2r&   N)r   r   r   rV   r  	STARCODERrE   r   r%   r&   r'   r	  r	  J  s    **J
3r&   r	  GPTRefactForCausalLMc                  X     e Zd Zej                  j
                  Z fdZd ZddZ	 xZ
S )RefactModelc                   t         |           t        j                  | j                  dg d      }|j                  dd       |j                  dd       |j                  dd	       d |_        |j                  | j                         y )
NFrv  r   middleeotr  special_token_typesrv  r   r   r   r  r   )	superr   rV   r  r.   r  chat_templater  r?   )rl   r  	__class__s     r'   r   zRefactModel.set_vocab_  sx     ))$..e@eg((15((15((15&*#!!$"2"23r&   c                   | j                   d   }d|z  }t        d|z  dz        }d}|||z   dz
  |z  z  }| j                   d   }| j                  j                  | j                   d          | j                  j	                  | j                   d          | j                  j                  |       | j                  j                  |       | j                  j                  | j                   d	          | j                  j                  d       | j                  j                  | j                   d
          | j                  j                  | j                         y )Nr   r   r   r      r   rK   r  r   r   )r:   r;   r?   r   r   r   r   r   r   r   r   r0   )rl   
hidden_dim	inner_dimmultiple_offf_dimr<   s         r'   r   zRefactModel.set_gguf_parametersk  s#   \\(+

N	Y*+
k!9A!=+ MNll9- 	++DLL,GH--dll8.DE008((5''X(>?**1-//=Q0RS&&tzz2r&   c                   | j                   d   }d|z  }t        d|z  dz        }d}|||z   dz
  |z  z  }| j                   d   }d}	| j                   d   |z  }
g }|V|d| d	k(  r|j                  | j                  t        j
                  j                  |      |d |	|
z   f       |j                  | j                  t        j
                  j                  |      ||	|
z  d  f       n|d| d
k(  r<|j                  | j                  t        j
                  j                  |      |f       n|d| dk(  r||j                  | j                  t        j
                  j                  |      |d | f       |j                  | j                  t        j
                  j                  |      ||d  f       t        |      dk(  r"|j                  | j                  |      |f       |S )Nr   r   r   r   r  r   r   ztransformer.h.z.attn.kv.weightz.attn.q.weightz.mlp.gate_up_proj.weightr   )r:   r;   rt  r   rV   r  r  r  r  FFN_GATEFFN_UPr[   r   )rl   r   r   r   r  r  r  r  r   r   r   rn  s               r'   r   zRefactModel.modify_tensors  s   \\(+

N	Y*+
k!9A!=+ MNh'	<<)V3,.?uO<< 7 78I8I8P8PRU VXbcxdmpxdxXyz{ 7 78I8I8P8PRU VXbclowcwcxXyz{>#n== 7 78I8I8P8PRU VXbcd>#.FGG 7 78I8I8R8RTW XZdelflZmno 7 78I8I8P8PRU VXbcicjXklmw<1NND006
CDr&   rf  )r   r   r   rV   r  REFACTrE   r   r   r   __classcell__r  s   @r'   r  r  [  s!    ''J
43(r&   r  StableLmForCausalLMStableLMEpochForCausalLMLlavaStableLMEpochForCausalLMc                       e Zd ZU ej                  j
                  Zd Zd ZdZ	de
d<   dZde
d<   d
dZdddZ fd	Z xZS )StableLMModelc                    | j                   dz  j                         r| j                          y | j                          y )Nr  )r.   r   r   r  r   s    r'   r   zStableLMModel.set_vocab  s2    NN--668  "   "r&   c                   | j                   }|d   }| j                  j                  |d          | j                  j                  |d          | j                  j	                  |       | j                  j                  |d          | j                  ddg      }| j                  j                  t        ||d   |d   z  z               | j                  j                  |d          | j                  j                  |d          | j                  j                  d	|v r|d	   nd
       | j                  j                  | j                  ddg             | j                  j                  | j                         y )NrJ   r   r   r   partial_rotary_factorrope_pctr   r   rt  Tr   norm_eps)r:   r?   r   r   r   r   r]   ru  r;   r   r   rv  r   r   r0   )rl   r:   r<   rotary_factors       r'   r   z!StableLMModel.set_gguf_parameters  s`   ,,12++G4M,NO--gm.DE((5009L1MN((*A:)NO11#mw}G]ahi~aG  7A  3B  	C''0E(FG**73H+IJ..SjnuSuw7N/O{  	A++D,<,<>NPZ=[,\]&&tzz2r&   Nlist[dict[str, Tensor]] | None_q_norms_k_normsc                   | j                   d   }| j                   d   }|j                  d      dk7  r|J | j                  (t        | j                        D cg c]  }i  c}| _        || j                  |   |<   t        | j                  |         |k\  r!| j                  ||| j                  |   d      S g S |j                  d      dk7  r|J | j                  (t        | j                        D cg c]  }i  c}| _        || j                  |   |<   t        | j                  |         |k\  r!| j                  ||| j                  |   d      S g S | j                  |      |fgS c c}w c c}w )Nr   r   zq_layernorm.normsr>  q_layernormzk_layernorm.normsk_layernorm)	r:   findr1  r  r<   r[   _stack_qk_normr2  r   )rl   r   r   r   r   r  rn   s          r'   r   zStableLMModel.modify_tensors  sg   34LL!67	99()R/?"?}}$-243C3C-D E E'1DMM#t$4==%&&0**3c8JMZZ	99()R/?"?}}$-243C3C-D E E'1DMM#t$4==%&)3**3	4==;M}]]	%%d+Z899- !F !Fs   	E+6	E0c                    g }t        |      D ]%  }d| d| d| d}|j                  ||          ||= ' t        j                  |d      }d| d| d}	| j	                  |	      }
|
|fgS )Nr  z.self_attn.z.norms.r  r   rz  )r  rt  rd   stackr   )rl   r   r   norms
layer_namedatasxidenamer   merged_namer   s              r'   r7  zStableLMModel._stack_qk_norm  s     = 	C#C5J<wse7SELLu&e	 [[A.
%cU+j\I''4:&''r&   c                   t         |           | j                  | j                  | j                  1| j                  D cg c]  }|j	                         D ]  }|  c}}ng | j                  1| j                  D cg c]  }|j	                         D ]  }|  c}}ng z   }t        |      dkD  rt        d|       y y c c}}w c c}}w )Nr   zUnprocessed norms: )r  rB  r1  r2  r|   r[   r   )rl   dry   r:  r  s       r'   rB  zStableLMModel.prepare_tensors  s    !==$(A AE@YDMM<q1668<a<<_a@D@YDMM<q1668<a<<_aE
 5zA~ #6ug!>??  )B =<s   CCrf  )r4  )r   r;   r   r;   r:  zdict[str, Tensor]r;  r   )r   r   r   rV   r  STABLELMrE   r   r   r1  rl  r2  r   r7  rB  r#  r$  s   @r'   r)  r)    sK    ))J#3  04H,3/3H,3:@(@ @r&   r)  LLaMAForCausalLMLlamaForCausalLMMistralForCausalLMMixtralForCausalLMc                       e Zd ZU ej                  j
                  Zd Z fdZe	d
d       Z
dZded<   ddZddZ fd	Z xZS )
LlamaModelc                h   	 | j                          | j
                  j                  dd      dk(  rt        j                  | j                  dg d      }|j                  dd       |j                  d	d
       |j                  dd       |j                  dd       |j                  | j                         | j                  dz  }|j                         rNt        |dd      5 }t        j                   |      }d|v r| j                  j#                  |d          d d d        | j
                  j                  dd      dk(  r| j                  j%                  d       y y # t        $ r; 	 | j                          n%# t        t        f$ r | j	                          Y nw xY wY w xY w# 1 sw Y   xY w)Nr  i }  i}  Fr  r  rv  i}  r   i}  r  i	}  r  i
}  r  r   r   r   add_prefix_spacei   )r  r#  r:  rU   r   r:   r   rV   r  r.   r  r  r?   r   r   r   r   add_add_space_prefixrX  )rl   r  r3  r   r4  s        r'   r   zLlamaModel.set_vocab  s   	'))+ <<L%0E9 --E&KM ,,Xu=,,Xu=,,Xu=,,Uu=%%d&6&67 $1H H ((*+S7C eq(,		!%%)>>$$99:OPb:cde <<L%0E9..u5 :7 ! 	''((*%y1 '$$&'	'*e esA   E! %8F(!	F%+E<;F%<FF%FF%$F%(F1c                2   t         |           | j                  }| j                  j	                  |d          d|v r|d   }n|d   |d   z  }| j                  j                  |       | j                  j                  d      d| j                  d   v r| j                  d   j                  d      dk(  r_| j                  j                  t        j                  j                         | j                  j                  | j                  d   d          y y y y )	Nr  r   r   r   r  r  rT   r  )r  r   r:   r?   add_vocab_sizeru  r   r  rV   r  r  r  )rl   r:   rope_dimr  s      r'   r   zLlamaModel.set_gguf_parameters  s    #%,,''(=> z*H}-9N1OOH11(;<<N+7HUcHd<d||N+//78C  66t7K7K7R7RS  88n9UV^9_` D =e7r&   c                    |||k7  r|} | j                   |d| j                  d   |z  dz  g| j                  dd   j                  dd      j                  | j                        S r  r  )r  r   r   s      r'   permutezLlamaModel.permute*  si     Vy%8F7==+;v+E+J_W]][\[]M^_!Q'	)r&   Nr0  _expertsc           	        | j                   d   }| j                   j                  d      }|j                  d      rt        j	                  |||      }|j                  d      rt        j	                  |||      }|j                  d      dk7  r| j                   d   }|J | j                  (t        | j                        D cg c]  }i  c}| _        || j                  |   |<   t        | j                  |         |dz  k\  rg }d	D ]  }	g }
t        |      D ]?  }d
| d| d|	 d}|
j                  | j                  |   |          | j                  |   |= A t        j                  |
d      }d| d|	 d}| j                  |      }|j                  ||f        |S g S | j                  |      |fgS c c}w )Nr   r   r  zq_proj.biasr  zk_proj.biasblock_sparse_moe.expertsr>  r   r   w1w2w3r  .block_sparse_moe.experts.r  r  r   rz  layers..feed_forward.experts.r:   r   r  rH  rP  r6  rQ  r  r<   r[   rt  rd   r9  r   rl   r   r   r   r   r  r   rn   rn  widr<  r=  r>  r?  r   s                  r'   r   zLlamaModel.modify_tensors4  s   34LL$$%:;	==9:#++JGJ==9:#++J	JJ 99/0B6%89I?"?}}$-243C3C-D E E'1DMM#t$4==%&)a-746 . ;C*,E$Y/ 6"/u4NseSTUXTYY` aT]]3%7%>? MM#.u56
 "'U!:J$+C50Fse7"SK#33K@HNNHj#9:; 	%%d+Z8997 !F   	F;c              #    K   | j                  dgd      x}r|j                  dd      j                         dk(  r| j                  j                  dd      }| j                  j                  d	| j                  d
   | j                  d   z        }d|t	        j
                  d|dt        j                        |z  z  z  }|j                  dd      }|j                  dd      }|j                  dd      }| j                  j                  dd      }||z  }	||z  }
|	|
k7  sJ g }|D ]q  }dt        j                  z  |z  }||
k  r|j                  d       0||	kD  r|j                  |       G||z  |z
  ||z
  z  }|j                  dd|z
  |z  |z   z         s | j                  t        j                  j                        t	        j                  |t        j                        f y y y wNr  Tr   	rope_typer  llama3r        @r   r   r         ?r   r   rc   r         @low_freq_factorhigh_freq_factorg      @ original_max_position_embeddingsi    r   r]   r   lowerr:   rd   aranger  mathpirt  r   rV   r  
ROPE_FREQStensorrl   r  baser{  freqsr  ri  rj  old_context_lenlow_freq_wavelenhigh_freq_wavelenrope_factorsfreqwavelensmooths                  r'   r   z!LlamaModel.generate_extra_tensorsa      ++^,<t+LL<LR0668HD||''g>ll&&z4<<3NRVR^R^_tRu3uvtQQemm(TWZ(Z[\%))(C8"."2"23Dc"J#/#3#34F#L "&,,"2"23UW["\#2_#D $36F$F!'+<<<<!! RD$''kD0G!22$++A. #33$++F3"1G";o"MRbetRt!u$++A!f*1F1O,PQR ..t/@/@/K/KLell[goto|o|N}~~3 E M   G/G1c                    t         |           | j                  M| j                  D cg c]  }|j                         D ]  }|  }}}t	        |      dkD  rt        d|       y y c c}}w Nr   zUnprocessed experts: r  rB  rQ  r|   r[   r   rl   rA  ry   expertsr  s       r'   rB  zLlamaModel.prepare_tensors~  m    !==$"&--BQBAqBqBGB7|a #8	!BCC   %B   A*)r  r   r   r;   r   rb  rf  ri  )r   r   r   rV   r  LLAMArE   r   r   rn  rP  rQ  rl  r   r   rB  r#  r$  s   @r'   rH  rH    sW    &&J6Ba  ) ) 04H,3+:Z:D Dr&   rH  BitnetForCausalLMc                  `     e Zd Zej                  j
                  Zd Z fdZddZ	ddZ
 xZS )BitnetModelc                $    | j                          y rw   r  r   s    r'   r   zBitnetModel.set_vocab  r  r&   c                    t         |           | j                  j                  t        j
                  j                         | j                  j                  d       y )Nrf  )r  r   r?   r  rV   r  r  r  rl   r  s    r'   r   zBitnetModel.set_gguf_parameters  sB    #%..t/C/C/J/JK005r&   c                   |j                   }|j                         }|j                         j                         j	                  d      }d|z  }||z  j                         j	                  dd      |z  }|j                  |      S )Nr  )minr   r>  )rc   floatabsmeanclamproundrT   )rl   weightrc   scaleiscaleresults         r'   weight_quantzBitnetModel.weight_quant  sv    

!!#))d)3U 6/((*00Q7&@{{5!!r&   c           
   #     K    j                  |      t         fdt        j                  j                  t        j                  j
                  t        j                  j                  t        j                  j                  t        j                  j                  t        j                  j                  t        j                  j                  fD              r j                  |      }|f y w)Nc              3  D   K   | ]  }j                  |        y wrw   r  r  s     r'   rz   z-BitnetModel.modify_tensors.<locals>.<genexpr>  s$      Ct++Hc3? r  )r   r  rV   r  r  r  r  ATTN_OUTr!  FFN_DOWNr   r  r  s   `  `@r'   r   zBitnetModel.modify_tensors  s     ''- $$$$$$&&$$&&&&L
   **:6J$$s   C/C4)r  r   r  r   rf  )r   r   r   rV   r  BITNETrE   r   r   r  r   r#  r$  s   @r'   r  r    s&    ''J(6
	"%r&   r  GrokForCausalLMc                  r     e Zd ZU ej                  j
                  Zd Z fdZ fdZ	dZ
ded<   ddZ xZS )		GrokModelc                $    | j                          y rw   r  r   s    r'   r   zGrokModel.set_vocab  r  r&   c                $    t        |   |i | y rw   )r  rp   rl   argskwargsr  s      r'   rp   zGrokModel.__init__  s    $)&)r&   c                "    t         |           y rw   )r  r   r  s    r'   r   zGrokModel.set_gguf_parameters  s    #%r&   Nr0  rQ  c           	        |j                  d      dk7  r| j                  d   }|J | j                  (t        | j                        D cg c]  }i  c}| _        || j                  |   |<   t        | j                  |         |dz  k\  rg }dD ]  }g }t        |      D ]?  }	d| d|	 d| d}
|j                  | j                  |   |
          | j                  |   |
= A t        j                  |d	
      }d| d| d}| j                  |      }|j                  ||f        |S g S | j                  |      |fgS c c}w )Nz.moe.r>  r   r   )r  linear_1linear_vztransformer.decoder_layer.r  r  r   rz  
r6  r:   rQ  r  r<   r[   rt  rd   r9  r   )rl   r   r   r   r   rn   rn  r_  r<  r=  r>  r?  r   s                r'   r   zGrokModel.modify_tensors  sg   99W#%89I?"?}}$-243C3C-D E E'1DMM#t$4==%&)a-746 > ;C*,E$Y/ 6"<SEse1SEQX YT]]3%7%>? MM#.u56
 "'U!:J$>se5W"UK#33K@HNNHj#9:; 	%%d+Z8997 !F   	Erf  )r   r   r   rV   r  GROKrE   r   rp   r   rQ  rl  r   r#  r$  s   @r'   r  r    s3    %%J(*& 04H,3#:r&   r  DbrxForCausalLMc                  N    e Zd Zej                  j
                  Zd ZddZddZ	y)	DbrxModelc                   | j                   d   }| j                   d   }| j                  j                  | j                   d          | j                  j                  | j                   d          | j                  j	                  | j                   d          | j                  j                  |d          | j                  j                  | j                   d          | j                  j                  |d          | j                  j                  |d	          | j                  j                  |d
          | j                  j                  |d          | j                  j                  |d          | j                  j                  d       | j                  j                  | j                         t        j!                  d| j                          y )N
ffn_configr  rI   r  r  ffn_hidden_sizer  r  r   r  moe_num_experts	moe_top_kr  r   )r:   r?   r   r   r   r   r   r   r   r  r   r   r   r   r0   rf   rg   )rl   r  r  s      r'   r   zDbrxModel.set_gguf_parameters  sn   \\,/
ll=1((j)AB++DLL,GH--dll9.EF00<M1NO''Y(?@**;|+DE++K,EF&&{:'>?))*5F*GH..z+/FG++D1&&tzz2(56r&   c                   ~| j                   d   d   }| j                   d   d   }| j                   d   }d dd d}d}|j                         D ]X  }	|j                  |	      dk7  s|j                  d	      dk(  s-d
}|j                  |||      }||	   x}
 |j                  |
 } n | j                  |s|n|d	z   d      }||fgS )Nr  r  r  r  )r   r   r   )zffn.experts.mlp.w1zffn.experts.mlp.w2zffn.experts.mlp.v1Fr>  r  Trc  r  )r:   r|   r6  r  rP  r   )rl   r   r   r   n_expertr   r   exp_tensor_namesr  exp_tensor_namepermute_tensorr   s               r'   r   zDbrxModel.modify_tensors  s    <<-.?@||L)*;<i( 372;268 /446 	Oyy)R/DIIi4HB4N'__XtVD
&6&GGNT!3!3!3^!DJ	 ''G	AQ`l'm:&''r&   c                    ~~~|dkD  S )Nr   r%   r   s        r'   r   zDbrxModel.tensor_force_quant%  s    (Czr&   Nrf  rh  )
r   r   r   rV   r  DBRXrE   r   r   r   r%   r&   r'   r  r    s!    %%J70!(Fr&   r  MiniCPMForCausalLMc                  `     e Zd Zej                  j
                  Z fdZddZd Z	ddZ
 xZS )MiniCPMModelc                H   t         |           t        | j                  d         }| j                  j                  |       t        j                  d|        | j                  d   | j                  d   dz  z  }| j                  j                  |       t        j                  d|        | j                  d   | j                  d   z  }| j                  j                  |       t        j                  d	|        | j                  j                  d
      | j                  d
   j                  d      dk(  rd| j                  j                  t        j                  j                         t        j                  dt        j                  j                          y y y )N	scale_embz"gguf: (minicpm) embedding_scale = scale_depthrJ   g      ?z!gguf: (minicpm) residual_scale = r   dim_model_basezgguf: (minicpm) logit_scale = r  rT   longropez$gguf: (minicpm) rope_scaling_type = )r  r   r  r:   r?   add_embedding_scalerf   rg   add_residual_scaleadd_logit_scaler   r  rV   r  LONGROPE)rl   embedding_scaleresidual_scalelogit_scaler  s       r'   r   z MiniCPMModel.set_gguf_parameters/  sX   #%[ 9:,,_=88IJKm4t||DW7X\_7__++N;77GHIll=1DLLAQ4RR((54[MBC<<N+7||N+//7:E  66t7K7K7T7TUB4CWCWC`C`Babc F 8r&   c              #    K   | j                   d   | j                   d   z  }| j                  dgd      }||j                  dd       }|j                  dd       }||t        d      t	        |      t	        |      k7  st	        |      |dz  k7  rt        d	|dz         | j                  t        j                  j                        t        j                  |t        j                  
      f | j                  t        j                  j                        t        j                  |t        j                  
      f y y w)Nr   r   r  Tlong_factorshort_factorNMissing the required key rope_scaling.long_factor or rope_scaling_short_factorr   2The length of rope long and short factors must be rg  )r:   r]   r   r{   r[   r   r   rV   r  ROPE_FACTORS_LONGrd   rr  r  ROPE_FACTORS_SHORT)rl   	rope_dimsr  long_factorsshort_factorss        r'   r   z#MiniCPMModel.generate_extra_tensors?  sH    LL/4<<@U3VV	''(8$?#'++M4@L(,,^TBM#}'<opp< C$66#l:Ky[\}:\ #UV_bcVcUd!eff**4+<+<+N+NOQVQ]Q]^jrwrr  RA  B  B**4+<+<+O+OPRWR^R^_lty  uB  uB  SC  D  D $s   EEc                $    | j                          y rw   r  r   s    r'   r   zMiniCPMModel.set_vocabP  r  r&   c                    ~| j                   d   }| j                   j                  d      }|j                  d      rt        j	                  |||      }|j                  d      rt        j	                  |||      }| j                  |      |fgS r  r:   r   r  rH  rP  r   rl   r   r   r   r   r  s         r'   r   zMiniCPMModel.modify_tensorsS  s    34LL$$%:;	 ==/+#++JGJ==/+#++J	JJ%%d+Z899r&   ri  rf  )r   r   r   rV   r  MINICPMrE   r   r   r   r   r#  r$  s   @r'   r  r  +  s(    ((Jd D"(:r&   r  MiniCPM3ForCausalLMc                  V    e Zd Zej                  j
                  Zd ZddZd Z	dddZ
y)	MiniCPM3Modelc                b   | j                   }| j                  j                  | j                         | j                  j	                  |d          | j                  j                  |d          | j                  j                  | j                         | j                  j                  |d          | j                  j                  |d          | j                  j                  |d          | j                  j                  |d          | j                  j                  |d          d|v r#|d   | j                  j                  |d          | j                  j                  |d	          | j                  j                  |d
   |d   z          | j                  j!                  |d          y )Nr   r   r   r   r   r   r  q_lora_rankkv_lora_rankqk_nope_head_dimqk_rope_head_dim)r:   r?   r   r0   r   r   r   r<   r   r   r   r   rM  add_q_lora_rankadd_kv_lora_rankr   ru  )rl   r:   s     r'   r   z!MiniCPM3Model.set_gguf_parametersf  ss   ,,&&tzz2++G4M,NO--gm.DE(()9)9:009L1MN''0E(FG**73H+IJ//0GH''(=>G#(>(J,,W]-CD))'.*AB''0B(CgN`Fa(ab11':L2MNr&   c              #  p  K   | j                  dgd      }|| j                  d   }|j                  dd       }|j                  dd       }||t        d      t	        |      t	        |      k7  st	        |      |dz  k7  rt        d|dz         | j                  t        j                  j                        t        j                  |t        j                  	      f | j                  t        j                  j                        t        j                  |t        j                  	      f y y w)
Nr  Tr  r  r  r  r   r  rg  )r]   r:   r   r{   r[   r   r   rV   r  r  rd   rr  r  r  )rl   r  r  r  r  s        r'   r   z$MiniCPM3Model.generate_extra_tensorsx  s:    ''(8$?#%78I'++M4@L(,,^TBM#}'<opp< C$66#l:Ky[\}:\ #UV_bcVcUd!eff**4+<+<+N+NOQVQ]Q]^jrwrr  RA  B  B**4+<+<+O+OPRWR^R^_lty  uB  uB  SC  D  D $s   D4D6c                $    | j                          y rw   r  r   s    r'   r   zMiniCPM3Model.set_vocab  r  r&   Nc                    |
||k7  r||z  } |j                   |d|j                  d   |z  dz  g|j                  dd   j                  dd      j                  |j                        S r  r  r  s       r'   r  z!MiniCPM3Model._reverse_hf_permute  r  r&   ri  rw   r  )r   r   r   rV   r  MINICPM3rE   r   r   r   r  r%   r&   r'   r  r  b  s'    ))JO$D"(
r&   r  QWenLMHeadModelc                  h    e Zd Zej                  j
                  Zed        Zeddd       Z	d Z
d Zy)r  c           	         ddl m}  |       }dj                  | j                  d      D cg c]  }|t	        |          c}      S c c}w Nr   )bytes_to_unicoder  zlatin-1*transformers.models.gpt2.tokenization_gpt2r  r9  r  ordbr  byte_encoderchars       r'   r  zQwenModel.token_bytes_to_string  <    O')wwAHHY<OPDSY/PQQP   ANc                >   |D cg c]  }t        |g       }}	 d }d }t        t        |d d |dd              D ].  \  }}| j                  |d   |d   z         }	|	#||	|k  s+|}|	}0 ||||k\  r	 |S |J |d | ||   ||dz      z   gz   ||dz   d  z   }c c}w Nr>  r   r   r   r  	enumeratezipr   
r  r  r  r  rG  min_idxmin_rankr  pairr  s
             r'   r  zQwenModel.bpe      %*+s++GH$SsU12Y%?@ $4&**47T!W+<=#)9TH_G#H	$
 H$8X=Q  &&&(7OuW~gk8J'J&KKeT[^_T_T`NaaE  ,   Bc                $    | j                          y rw   )r  r   s    r'   r   zQwenModel.set_vocab  r   r&   c                   | j                   j                  | j                  d          | j                   j                  | j                  d          | j                   j	                  | j                  d          | j                   j                  | j                  d          | j                   j                  | j                  d          | j                   j                  | j                  d   | j                  d   z         | j                   j                  | j                  d          | j                   j                  | j                  d          | j                   j                  | j                         y )Nr   rJ   r   r   rotary_emb_baser   r   )r?   r   r:   r   r   r   r   ru  r   r   r   r0   r   s    r'   r   zQwenModel.set_gguf_parameters  s   ++DLL9R,ST((6I)JK--dll=.IJ00>Q1RS++DLL9J,KL11$,,}2MQUQ]Q]^sQt2tu''5J(KL//=Q0RS&&tzz2r&   rw   r  zdict[bytes, int]r  r  r  rb  r  zlist[bytes])r   r   r   rV   r  QWENrE   rn  r  r  r   r   r%   r&   r'   r  r    sC    %%JR R
   	3r&   r  Qwen2ForCausalLMc                  P     e Zd Zej                  j
                  Zd Z fdZ xZ	S )
Qwen2Modelc                d    	 | j                          y # t        $ r | j                          Y y w xY wrw   )r  r#  r   r   s    r'   r   zQwen2Model.set_vocab  s.    	#))+  	#  "	#s    //c                   t         |           | j                  j                  d      d| j                  d   v r| j                  d   j                  d      dk(  r| j                  j                  t        j                  j                         | j                  j                  | j                  d   d          | j                  j                  | j                  d   d          y y y y )Nr  r  rT   yarnrk  )r  r   r:   r   r?   r  rV   r  YARNr  add_rope_scaling_orig_ctx_lenr  s    r'   r   zQwen2Model.set_gguf_parameters  s    #%<<N+7HUcHd<d||N+//76A  66t7K7K7P7PQ  88n9UV^9_`  >>t||N?[\~?  A B =e7r&   )
r   r   r   rV   r  QWEN2rE   r   r   r#  r$  s   @r'   r  r    s#    &&J#A Ar&   r  Qwen2MoeForCausalLMc                  l     e Zd ZU ej                  j
                  Z fdZdZde	d<   ddZ
 fdZ xZS )Qwen2MoeModelc                   t         |           | j                  j                  d      x}| j                  j                  |       | j                  j                  d      x}3| j                  j                  |       t        j                  d|        | j                  j                  d      x}4| j                  j                  |       t        j                  d|        y y )Nnum_expertsmoe_intermediate_sizez#gguf: expert feed forward length = shared_expert_intermediate_sizez*gguf: expert shared feed forward length = )
r  r   r:   r   r?   r   add_expert_feed_forward_lengthrf   rg   %add_expert_shared_feed_forward_length)rl   r   r  r  r  s       r'   r   z!Qwen2MoeModel.set_gguf_parameters  s    #%))-88IE--i8%)\\%5%56M%NN![;;<QRKK=>S=TUV/3||/?/?@a/bb+oBBCbcKKDEdDefg pr&   Nr0  rQ  c           	        |j                  d      dk7  r| j                  d   }|J | j                  (t        | j                        D cg c]  }i  c}| _        || j                  |   |<   t        | j                  |         |dz  k\  rg }dD ]  }g }t        |      D ]?  }	d| d|	 d| d	}
|j                  | j                  |   |
          | j                  |   |
= A t        j                  |d
      }d| d| d	}| j                  |      }|j                  ||f        |S g S | j                  |      |fgS c c}w Nr  r>  r  r   	down_proj	gate_projup_projr  .mlp.experts.r  r  r   rz  r  rl   r   r   r   r   rn   rn  w_namer<  r=  r>  r?  r   s                r'   r   zQwen2MoeModel.modify_tensors  f   99Y2%]3I?"?}}$-243C3C-D E E'1DMM#t$4==%&)a-746 D ;F*,E$Y/ 6"/uM#axw WT]]3%7%>? MM#.u56
 "'U!:J$1#mF87"SK#33K@HNNHj#9:; 	%%d+Z8997 !Fr  c                    t         |           | j                  M| j                  D cg c]  }|j                         D ]  }|  }}}t	        |      dkD  rt        d|       y y c c}}w r  r  r  s       r'   rB  zQwen2MoeModel.prepare_tensors	  r  r  rf  )r   r   r   rV   r  QWEN2MOErE   r   rQ  rl  r   rB  r#  r$  s   @r'   r  r    s7    ))J	h 04H,3":HD Dr&   r  GPT2LMHeadModelc                  F    e Zd Zej                  j
                  Zd ZddZy)	GPT2Modelc                4   | j                   j                  | j                  d          | j                   j                  | j                  d          | j                   j	                  | j                  d          | j                   j                  d| j                  d   z         | j                   j                  | j                  d          | j                   j                  | j                  d          | j                   j                  | j                         y )NrK   r   r   r   r   r   
r?   r   r:   r   r   r   r   r   r   r0   r   s    r'   r   zGPT2Model.set_gguf_parameters  s    ((i)@A++DLL,AB--dll8.DE00T\\(5K1KL''X(>?++DLL9M,NO&&tzz2r&   c                   ~g }|j                  d      r|S |j                  d      r|j                  dd      }| j                  |      }|j                  ||f       || j	                  t
        j                  j                        k(  r:|j                  | j	                  t
        j                  j                        |f       |S )N)
.attn.biasz.attn.masked_bias).c_attn.weight.c_proj.weight.c_fc.weightr,  r   r   )	r  	transposer   rt  r   rV   r  r'  r(  )rl   r   r   r   rn  r   s         r'   r   zGPT2Model.modify_tensors   s    ,. ==<=N==_`#--a3J''-*-. t..t/@/@/K/KLLNND33D4E4E4L4LMzZ[r&   Nrf  )	r   r   r   rV   r  GPT2rE   r   r   r%   r&   r'   r&  r&    s    %%J3r&   r&  PhiForCausalLMc                  >    e Zd Zej                  j
                  Zd Zy)	Phi2Modelc                2   | j                  ddg      }| j                  dg      }| j                  ddg      }| j                  ddg      }| j                  j                  | j                  dd	g             | j                  j                  |       | j                  j	                  d
|z         | j                  j                  |       | j                  j                  |       | j                  j                  |       | j                  j                  | j                  ddg             | j                  j                  t        ||z        |z         | j                  j                  | j                         | j                  j                  d       y )NrJ   rK   r,  r   r   r   r   r  r   r   r   r   F)r]   r?   r   r   r   r   r   r   r   ru  r;   r   r0   rX  )rl   r<   rot_pctr   r   s        r'   r   zPhi2Model.set_gguf_parameters;  sY   &&(;Y'GH""$;#<=!!=(";<!!#8("CD++D,<,<mMf=g,hi--f500V<((5''/**62++D,<,<>RTd=e,fg11#g6F2G62QR&&tzz2**51r&   N)r   r   r   rV   r  PHI2rE   r   r%   r&   r'   r2  r2  7  s    %%J2r&   r2  Phi3ForCausalLMc                  L    e Zd Zej                  j
                  Zd Zd ZddZ	y)Phi3MiniModelc                n   ddl m} | j                  dz  }|j                         st	        d|        |       }|j                  t        |             | j                  j                  d|j                               }t        |      D cg c]  }d| dj                  d       }}d	g|z  }t        j                  g|z  }t        |j                               D ]  }	|j                  |	      }
|
j                  d      }|j                  |	      }t        j                   }|j#                  |	      rt        j$                  }ne|j'                  |	      rt        j(                  }nC|j+                  |	      rt        j                  }n!|j-                  |	      rt        j.                  }|||	<   |||	<   |||	<    | j                  d
z  }|j                         rt1        |dd      5 }t3        j4                  |      }|D ]W  }||   }	|	|k\  rt6        j9                  d|	 d|dz
          ,|j                  d      ||	<   d||	<   t        j:                  ||	<   Y 	 d d d        | j                  dz  }|j                         rt1        |dd      5 }t3        j4                  |      }|j                  di       }|j=                         D ]  \  }	}t?        |	      }	|d   j                  d      }||	   t        j                  k7  rG||	   |k7  r?t6        jA                  d|	 d||	   jC                  d      d|jC                  d             |||	<   d||	<   t        j:                  ||	<   |j                  d      st        j(                  ||	<    	 d d d        | j                  dz  }|j                         rt1        |dd      5 }t3        j4                  |      }|j                  dg       }|D ]  }t?        |d         }	|d   j                  d      }||	   t        j                  k7  rG||	   |k7  r?t6        jA                  d|	 d||	   jC                  d      d|jC                  d             |||	<   d||	<   t        j:                  ||	<   |j                  d      st        j(                  ||	<    	 d d d        | jD                  jG                  d       | jD                  jI                  d       | jD                  jK                  |       | jD                  jM                  |       | jD                  jO                  |       tQ        jR                  | j                  tU        |            }|jW                  | jD                         y c c}w # 1 sw Y   8xY w# 1 sw Y   xY w# 1 sw Y   xY w)Nr   r  r  Error: Missing r  r  r  r   r  r  r   r   r  r  r   r  r  r  r  r  r  r  r  r  added_tokensidr	  r
  r  ),r"  r  r.   r   r   r$  r   r:   r   r  r  r  r   r#   r%  r&  r   r'  r    r(  r!   r)  r*  r$   r   r   r   rf   r  r"   r  r;   r5  r  r?   r  r  r  r  r  rV   r  r[   r  )rl   r  r+  r  r  r  r  r  r  r,  r-  r.  r/  r0  r1  r   r2  r~   r3  r4  r  
foken_datar  tokenizer_filetokenizer_jsonr;  r  s                              r'   r   zPhi3MiniModel.set_vocabS  s   8*;;%%'~.>?@@*,	s>23\\%%lI4H4H4JK
DI*DUVqaS{11':VV'j:56==>Ki2245 	)H''1E<<(D&&x0E-44G""8,199$$X.199##H-188!!(+166#F8$F8!(HX%	)( !NN-@@$$&'w? N1$(IIaL!, NC05H:-}XJ>XYcfgYgXh%ij '*zz'':F8$'.F8$)@)M)MHX&NN !%1H H ((*+S7C Mq(,		!%'<'@'@AWY['\$,@,F,F,H 
M(Hj"8}H&y188AE)-D-K-KK!(+u4"NN-=hZr&QYJZJaJabiJjImmqrwr~r~  @G  sH  rK  ,L  M',F8$'.F8$)@)M)MHX&!~~i0-D-L-L*
MM *::!!#ncG< M!%1-11."E". 
MJ":d#34H&y188AE)-D-K-KK!(+u4"NN-=hZr&QYJZJaJabiJjImmqrwr~r~  @G  sH  rK  ,L  M',F8$'.F8$)@)M)MHX&!~~i0-D-L-L*
MM 	,,W5**95''/))&1((2))$..#f+N!!$"2"23a W4N NM M"M Ms9   V(A2VC(V:VCV+!V+VV(+V4c                   | j                  ddg      }| j                  ddg      }| j                  ddg      }| j                  ddg      }| j                  d	g      }| j                  d
dg      }| j                  dg      }||z  }| j                  j                  |       | j                  j                  |       | j                  j	                  |       | j                  j                  | j                  dg             | j                  j                  |       | j                  j                  |       | j                  j                  |       | j                  j                  |       | j                  j                  |       | j                  j                  | j                  dg             | j                  j                  | j                         | j                  j                  | j                  dg             y )NrJ   rK   r   r   r   r   r   r   r   r  r   rk  r   r   sliding_window)r]   r?   r   r  r   r   r   r   r   r   ru  r   r   r0   add_sliding_window)	rl   r<   r   r   r   rms_epsmax_pos_embdsorig_max_pos_embdsr  s	            r'   r   z!Phi3MiniModel.set_gguf_parameters  s   &&(;Y'GH!!=(";<!!#8("CD$$&;[%IJ	""N#34((-9R)ST!--/Q.RSf$	++M:667IJ--f5001A1ACVBW1XY((5''/**95//811)<++D,<,<l^,LM&&tzz2++D,<,<>N=O,PQr&   c              #    K   | j                  ddg      }| j                  ddg      }| j                  ddg      }| j                  dg      }||z  }| j                  dgd	      }|y ||z  }|j                  d
d      j                         }t        |      dk(  rt	        d      |dk(  s|dk(  rI|dkD  rAt        j                  dt        j                  |      t        j                  |      z  z         nd}	n7|dk(  r#|dkD  rdt        j                  |      z  dz   nd}	nt        d| d      | j                  j                  |	       |j                  dd       }
|j                  dd       }|
|t	        d      t        |
      t        |      k7  st        |
      |dz  k7  rt        d|dz         | j                  t        j                  j                        t!        j"                  |
t         j$                        f | j                  t        j                  j&                        t!        j"                  |t         j$                        f y w)Nr   r   r   r   r  r   rk  r  TrT   r  r   z*Missing the required key rope_scaling.typesur  rf  r   r  皙?zThe rope scaling type z is not supported yetr  r  r  r   r  rg  )r]   r   rm  r[   r{   ro  sqrtlogr  r?   add_rope_scaling_attn_factorsr   r   rV   r  r  rd   rr  r  r  )rl   r   r   rD  rE  r  r  r  rope_scaling_typeattn_factorr  r  s               r'   r   z$Phi3MiniModel.generate_extra_tensors  sB    !!=(";<!!#8("CD((-9R)ST!--/Q.RSf$	 ''(8$? 22(,,VR8>>@ !Q&GHH$(9Z(G[`cf[f$))A$((CU:V(V$VWloK&(9>#/#5#K%(>?P>QQf&ghh66{C#''t<$((>=#8kll|M 22c,6G9WX=6XQR[^_R_Q`abb&&t'8'8'J'JKU\\Zfnsn{n{M|}}&&t'8'8'K'KLell[hpup}p}N~s   IINri  )
r   r   r   rV   r  PHI3rE   r   r   r   r%   r&   r'   r8  r8  O  s$    %%J]4~R0%@r&   r8  PlamoForCausalLMc                  X    e Zd Zej                  j
                  Zd Zd Zd Z	d Z
ddZy)
PlamoModelc                $    | j                          y rw   r  r   s    r'   r   zPlamoModel.set_vocab  r  r&   c                   | j                   }|d   }| j                  j                  d       | j                  j                  |d          | j                  j	                  |d          | j                  j                  |       | j                  j                  |d          | j                  j                  d       | j                  j                  |d          | j                  j                  | j                         y )NrJ      r   r   r   r   r   )r:   r?   r   r   r   r   r   r   r   r   r0   rl   r:   r<   s      r'   r   zPlamoModel.set_gguf_parameters  s    ,,12++D1--gm.DE009L1MN((5''0E(FG**1-//0GH&&tzz2r&   c                    |j                         dk(  sJ |j                  dddd      }t        j                  |d      }t        j                  |d      }|S )N   rX     r      rX  )r   r   r   r   sizer  rd   rP  rl   r   s     r'   shuffle_attn_q_weightz PlamoModel.shuffle_attn_q_weight	  sS     L000''1c48
]]:|<
]]:|<
r&   c                    |j                         dk(  sJ |j                  dddd      }t        j                  |d      }t        j                  |d      }|S )NrW  rX  rY  r   rZ  )r   r   r   r   r[  r]  s     r'   shuffle_attn_output_weightz%PlamoModel.shuffle_attn_output_weight	  sS     L000''aC8
]]:|<
]]:|<
r&   c                    ~| j                  |      }|j                  d      r| j                  |      }n"|j                  d      r| j                  |      }||fgS )Nzattn_q.weightzattn_output.weight)r   r  r^  r`  r  s        r'   r   zPlamoModel.modify_tensors	  sb    ''- _-33J?J3488DJ:&''r&   Nrf  )r   r   r   rV   r  PLAMOrE   r   r   r^  r`  r   r%   r&   r'   rQ  rQ    s*    &&J(3(r&   rQ  CodeShellForCausalLMc                  F    e Zd Zej                  j
                  Zd ZddZy)CodeShellModelc                Z   | j                   d   }| j                  j                  | j                   d          | j                  j                  | j                   d          | j                  j	                  d| j                   d   z         | j                  j                  |       | j                  j                  | j                   d          | j                  j                  | j                   d          | j                  j                  | j                   d          | j                  j                  | j                         | j                  j                  d       | j                  j                  t        j                  j                         | j                  j!                  d	       y )
NrK   r  r   r   r   num_query_groupsr   re  rf  )r:   r?   r   r   r   r   r   r   r   r   r0   r   r  rV   r  r  r  rw  s     r'   r   z"CodeShellModel.set_gguf_parameters&	  s9   ll9-++DLL,GH--dll8.DE00T\\(5K1KL((5''X(>?**4<<8J+KL++DLL9M,NO&&tzz2++G4..t/C/C/J/JK005r&   c                F    ~ j                  |      }||fg}| j                  t        j                  j                        k(  r\ j
                  J t         fddD              r:|j                   j                  t        j                  j                        |f       |S )Nc              3  :   K   | ]  }|j                   v  y wrw   r  r  s     r'   rz   z0CodeShellModel.modify_tensors.<locals>.<genexpr>?	  r  r  r  )	r   r   rV   r  r'  r>   r  rt  r(  )rl   r   r   r   r   rn  s   `     r'   r   zCodeShellModel.modify_tensors5	  s    ''-.6
-C,Dt..t/@/@/K/KLL$$000[7Z[[ 7 78I8I8P8P QS]^_r&   Nrf  )	r   r   r   rV   r  	CODESHELLrE   r   r   r%   r&   r'   re  re  "	  s    **J6r&   re  InternLM2ForCausalLMc                  L    e Zd Zej                  j
                  Zd Zd ZddZ	y)InternLM2Modelc                p   ddl m} ddl m} | j                  dz  }g }g }g }|j	                         s-t
        j                  d|        t        j                  d       |j                         }|j                  t        |d      j                                |j                  j                  } |       }	|	j                  t!        |             | j"                  j%                  d|	j'                               }
t)        |
      D ]P  }|	j+                  |      }|j-                  d	      }|	j/                  |      }|d
k(  r*t
        j1                  d| d       dj-                  d	      }t2        j4                  }|	j7                  |      rt2        j8                  }ne|	j;                  |      rt2        j<                  }nC|	j?                  |      rt2        j@                  }n!|	jC                  |      rt2        jD                  }|jG                  d      rt2        j@                  }|jI                  |       |jI                  |       |jI                  |       S | j                  dz  }|j	                         rt        |dd	      5 }tK        jL                  |      }|D ]R  }|jI                  |j-                  d	             |jI                  d       |jI                  t2        jN                         T 	 d d d        d}d }| j                  dz  }|j	                         rt        |dd	      5 }tK        jL                  |      }|j%                  di       }|jQ                         D ]  \  }}tS        |      }|d   }||k(  r|}|j-                  d	      }||   t2        j@                  k7  rG||   |k7  r?t
        j1                  d| d||   jU                  d	      d|jU                  d	             |||<   d||<   t2        jN                  ||<   |j%                  d      st2        j<                  ||<    	 d d d        | j                  dz  }|j	                         rt        |dd	      5 }tK        jL                  |      }|j%                  dg       }|D ]  }tS        |d         }|d   }||k(  r|}|j-                  d	      }||   t2        j@                  k7  rG||   |k7  r?t
        j1                  d| d||   jU                  d	      d|jU                  d	             |||<   d||<   t2        jN                  ||<   |j%                  d      st2        j<                  ||<    	 d d d        | jV                  jY                  d       | jV                  j[                  d       | jV                  j]                  |       | jV                  j_                  |       | jV                  ja                  |       | jV                  jc                  |       te        jf                  | j                  ti        |             }|jj                  d!   }|+||jj                  d!<   t
        j1                  d"| d#| d$       |jm                  | jV                         y # 1 sw Y   xY w# 1 sw Y   kxY w# 1 sw Y   HxY w)%Nr   r  sentencepiece_model_pb2r  r:  r   rbr  r   r  zInternLM2 convert token 'u   ' to '🐉'!u   🐉z[UNUSEDr  r   r   r  z
<|im_end|>r  r  r  r  r  r  r  r  r;  r<  r	  r
  r  r  zReplace eos:z with a special token:z8 in chat mode so that the conversation can end normally.)7r"  r  rp  r.   r   rf   errorr@  exit
ModelProtoParseFromStringr   readnormalizer_specadd_dummy_prefixr$  r   r:   r   r  r  r%  r  r&  r5  r   r   r'  r    r(  r!   r)  r#   r*  r$   rs  rt  r   r   r"   r  r;   r  r?   r  r  r  r  r  rK  rV   r  r[   r  r  ) rl   r  rG   r+  r  r  r  sentencepiece_model
add_prefixr  r  r,  r-  r.  r/  r0  r1  r   r2  r~   chat_eos_tokenchat_eos_token_idr3  r4  r  r=  r  r>  r?  r;  r  old_eoss                                    r'   r   zInternLM2Model.set_vocabJ	  sM   
 	9B*;;   %%'LL?>*:;<HHQK#..0++D,F,K,K,MN(88II
*,	s>23\\%%lI4H4H4JK
j) 	%H''1E<<(D&&x0Ew !:4&MN}}W--44G""8,199$$X.199##H-188!!(+166	*188MM$MM% OOG$3	%6 !NN-@@$$&'w? J1$(IIaL!, JCMM#**W"56MM'*OO$;$H$HIJJ &  $1H H ((*+S7C Mq(,		!%'<'@'@AWY['\$,@,F,F,H M(Hj"8}H&y1E.,4)!LL1E)-D-K-KK!(+u4"NN-=hZr&QYJZJaJabiJjImmqrwr~r~  @G  sH  rK  ,L  M',F8$'.F8$)@)M)MHX&!~~i0-D-L-L*MM$ *::!!#ncG< M!%1-11."E". MJ":d#34H&y1E.,4)!LL1E)-D-K-KK!(+u4"NN-=hZr&QYJZJaJabiJjImmqrwr~r~  @G  sH  rK  ,L  M',F8$'.F8$)@)M)MHX&!~~i0-D-L-L*MM$ 	,,W5**95''/))&1((2--j9))$..#f+N11%8( 6GM++E2NN\'2HIZH[UV W 	!!$"2"23IJ JM M(M Ms3   A-Z-C1ZZ+C#Z+Z+ZZ(+Z5c                *   | j                   j                  | j                  d          | j                   j                  | j                  d          | j                   j	                  | j                  d          | j                   j                  | j                  d          | j                   j                  | j                  d          | j                   j                  | j                  d          | j                   j                  | j                  d          | j                   j                  | j                  d          | j                   j                  | j                         | j                  j                  d	      d
| j                  d	   v r| j                  d	   j                  d      dk(  r_| j                   j                  t        j                  j                          | j                   j#                  | j                  d	   d
          y y y y )Nr   rJ   r   r   r   r   r   r   r  r  rT   r  )r?   r   r:   r   r   r   r   r   r   r   r   r0   r   r  rV   r  r  r  r   s    r'   r   z"InternLM2Model.set_gguf_parameters	  s   ++DLL9R,ST((6I)JK--dll=.IJ00>Q1RS++DLL,FG''5J(KL//^0LM**4<<8M+NO&&tzz2<<N+7HUcHd<d||N+//78C  66t7K7K7R7RS  88n9UV^9_` D =e7r&   c                B   | j                   d   }| j                   d   }| j                   d   }||z  }||z  }||z  }	|Md| d|v rD|}
|
j                  |	|dz   ||f      }
|
d d d |f   |
d d df   |
d d df   }}}t        j                  |j                  d|j                  d   f      ||      }t        j                  |j                  d|j                  d   f      ||      }|j                  d|j                  d   f      }| j                  t        j                  j                  |      |f| j                  t        j                  j                  |      |f| j                  t        j                  j                  |      |fgS | j                  |      |fgS )	Nr   r   r   r  z.attention.wqkvr   r   r>  )r:   r  rH  rP  r  r   rV   r  r  r  r  r   )rl   r   r   r   	num_headsr  r   q_per_kvr   
num_groupsr  r  ry   r  s                 r'   r   zInternLM2Model.modify_tensors	  s   LL!67	||$9:m,,Y&(*
?se?CtKC++z8a<6JKC!ZxZ-(#ae*c!R%j!qA ""199b!''"+->#?IVA""199b!''"+->#?LYA		2qwwr{+,A (():):)A)A3GK(():):)A)A3GK(():):)A)A3GK  ))$/<==r&   Nrf  )
r   r   r   rV   r  	INTERNLM2rE   r   r   r   r%   r&   r'   rm  rm  F	  s#    **J|4|a>r&   rm  	BertModelCamembertModelRobertaModelc                  b     e Zd Zej                  j
                  Z fdZ fdZd Z	ddZ
 xZS )r  c                2    t        |   |i | d | _        y rw   )r  rp   r  r  s      r'   rp   zBertModel.__init__	  s    $)&)r&   c                   t         |           | j                  j                  d       d }| j                  dz  }|j                         rAt        |d      5 }t        j                  |      }d d d        D ]  }|d   dk(  s|d   } n |t        | j                  |z  dz  d      5 }t        j                  |      }d d d        d	   rt        j                  j                  }n+|d
   rt        j                  j                  }nt        d      | j                  j                  |       y y # 1 sw Y   xY w# 1 sw Y   }xY w)NFzmodules.jsonr   r   rT   z$sentence_transformers.models.PoolingrN   ry  pooling_mode_mean_tokenspooling_mode_cls_tokenz)Only MEAN and CLS pooling types supported)r  r   r?   add_causal_attentionr.   r   r   r   r   rV   PoolingTypeMEANCLSr  add_pooling_type)	rl   pooling_pathmodule_pathr   modulesmodpoolingpooling_typer  s	           r'   r   zBertModel.set_gguf_parameters	  s7   #%--e4 nn~5 kG4 '))A,' v;"HH#&v;L #dnn|3mCgV 'Z[))A,'12#//4412#//33)*UVV--l; $' '' 's   D2,D>2D;>Ec                H   | j                         \  }}}t        |      | _        | j                  j	                  | j
                  j                  dd             d }t        t        ||            }| j                  j                  d       | j                  j                  |       | j                  j                  |       | j                  j                  |       t        j                  | j                  t        |            }|j!                  | j                         y )Ntype_vocab_sizer   c                    | j                  d      r| j                  d      r| S | j                  d      r| dd  S d| z   S )N[r  z##r   u   ▁)rs  r  )toks    r'   phantomz$BertModel.set_vocab.<locals>.phantom 
  s?    ~~c"s||C'8
~~d#12wc>!r&   bertr  )r  r[   r  r?   add_token_type_countr:   r   listr  r  r  r  r  rV   r  r.   r  )rl   r  r  r  r  r  s         r'   r   zBertModel.set_vocab
  s    #'#6#6#8 &f+
 	--dll.>.>?PRS.TU	" c'6*+ 	,,V4**62''/((2 ))$..#f+N!!$"2"23r&   c                8    ~|dv rg S | j                  |      |fgS )N)zembeddings.position_idszpooler.dense.weightzpooler.dense.biasr   r   s       r'   r   zBertModel.modify_tensors2
  s.     ZZI%%d+Z899r&   rf  )r   r   r   rV   r  BERTrE   rp   r   r   r   r#  r$  s   @r'   r  r  	  s&    %%J<648:r&   NomicBertModelc                  T     e Zd Zej                  j
                  Z fdZ fdZ xZ	S )r  c                   t        |   |i | d| j                  d<   | j                  d   dk(  sJ | j                  d   du sJ | j                  d   du sJ | j                  d   du sJ | j                  d	   du sJ | j                  d
   du sJ | j                  d   dk(  sJ | j                  d   du sJ | j                  d   J y )Nr  r   activation_functionswiglucausalFqkv_proj_biasmlp_fc1_biasmlp_fc2_biasprenormrotary_emb_fractionrf  rotary_emb_interleavedrotary_emb_scale_base)r  rp   r:   r  s      r'   rp   zNomicBertModel.__init__@
  s    $)&) !%W ||12h>>>||H%...||O,555||N+u444||N+u444||I&%///||12c999||45>>>||34<<<r&   c                r    t         |           | j                  j                  | j                  d          y )Nr  )r  r   r?   r   r:   r  s    r'   r   z"NomicBertModel.set_gguf_parametersU
  s,    #%++DLL9J,KLr&   )
r   r   r   rV   r  
NOMIC_BERTrE   rp   r   r#  r$  s   @r'   r  r  <
  s#    ++J=*M Mr&   XLMRobertaModel#XLMRobertaForSequenceClassificationc                  \     e Zd Zej                  j
                  Z fdZd Zd fdZ	 xZ
S )r  c                    t        |   |i | | j                  j                  d      x};d|z   | _        d| j                  v r"| j                  dxx   | j                  z  cc<   y y d | _        y )Npad_token_idr   r   )r  rp   r:   r   _position_offset)rl   r  r  r  r  s       r'   rp   zXLMRobertaModel.__init__^
  st    $)&) !LL,,^<<LI$%$4D!(DLL8674;P;PP7 9 %)D!r&   c                	   dt         j                  d<   ddlm} ddlm} | j
                  dz  }|j                         st        d|       |j                         }|j                  t        |d      j                                |j                  j                  d	k(  sJ |j                  j                  }|j                  j                   }|j                  j"                  } |       }|j%                  t'        |             | j(                  j+                  d
|j-                               }	t/        |	      D 
cg c]  }
d|
 dj1                  d       }}
dg|	z  }t2        j4                  g|	z  }t/        |j-                               D ]  }|j7                  |      }|j1                  d      }|j9                  |      }t2        j:                  }|j=                  |      rt2        j>                  }ne|jA                  |      rt2        jB                  }nC|jE                  |      rt2        j4                  }n!|jG                  |      rt2        jH                  }|||<   |||<   |||<    |	tK        |      kD  r|	tK        |      z
  }tL        jO                  d| d| d       t/        d	|d	z         D ]R  }
|jQ                  tS        d|
 dd             |jQ                  d       |jQ                  t2        j4                         T g d|dd z   }g d|dd z   }t2        jB                  t2        jB                  t2        jB                  t2        j>                  g|dd z   }| jT                  jW                  d       | jT                  jY                  d       | jT                  j[                  |       | jT                  j]                  |       | jT                  j_                  |       | jT                  ja                  |       | jT                  jc                  | j(                  j+                  dd	             | jT                  je                  |       |r| jT                  jg                  |       ti        jj                  | j
                  tK        |            }|jm                  | jT                         | jT                  jo                  d       | jT                  jq                  d       y c c}
w )Npython&PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATIONr   r  ro  zsentencepiece.bpe.modelr  rq  r   r  r  r  r   r  r   r!  r   r  )s   <s>s   <pad>s   </s>s   <unk>r   r>  )r  r  r  r  t5r
  r  r  T)9rq  environr"  r  rp  r.   r   r#  rt  ru  r   rv  trainer_specrJ  rw  rx  remove_extra_whitespacesprecompiled_charsmapr$  r   r:   r   r  r  r  r   r#   r%  r&  r   r'  r    r(  r!   r)  r*  r$   r[   rf   r  rt  r  r?   r  r  r  r  r  rK  r  add_remove_extra_whitespacesadd_precompiled_charsmaprV   r  r  rX  rZ  )rl   r  rG   r+  ry  rz  remove_whitespacesr  r  r  r  r  r  r  r,  r-  r.  r/  r0  r6  r  s                        r'   r   zXLMRobertaModel.set_vocabi
  s<    @H

;<8B*CC%%'#&6~6F$GHH#..0++D,F,K,K,MN"//::a???(88II
0@@YY2BBWW*,	s>23\\%%lI4H4H4JK
DI*DUVqaS{11':VV'j:56==>Ki2245 	)H''1E<<(D&&x0E-44G""8,199$$X.199##H-188!!(+166#F8$F8!(HX#	)& F#"S[0ILL.yk9XYbXccdef1i!m, @ed1#QK'BCg& 7 > >?@ 7"E%q4#++#++#++#++	

 QrN 	,,T2**95''/))&1((2--j9--dll.>.>?PRS.TU556HI556JK))$..#f+N!!$"2"23**40**40q Ws   ;S)c                    |j                  d      r|dd  }|dk(  r!| j                  || j                  d d d f   }t        |   |||      S )Nzroberta.rY  z%embeddings.position_embeddings.weight)rs  r  r  r   rl   r   r   r   r  s       r'   r   zXLMRobertaModel.modify_tensors
  s_     ??:&8D ::$$0'(=(=(>q(@A
w%j$<<r&   rf  )r   r   r   rV   r  r  rE   rp   r   r   r#  r$  s   @r'   r  r  Z
  s(    %%J	)P1d= =r&   GemmaForCausalLMc                  L    e Zd Zej                  j
                  Zd Zd ZddZ	y)
GemmaModelc                   | j                          t        j                  | j                  dg d      }|j	                  dd       |j	                  dd       |j	                  dd	       |j	                  d
d       |j	                  dd       d |_        |j                  | j                         | j                  j                  d       y )NF)rv  r   r  fsepr  r  rv  C   r   E   r  D   r  F   r  k   )	r  rV   r  r.   r  r  r  r?   rK  )rl   r  s     r'   r   zGemmaModel.set_vocab
  s    %%' ))$..e@mo((26((26((26((26((37&*#!!$"2"23--e4r&   c                   | j                   }|d   }| j                  j                  |d          | j                  j                  |d          | j                  j	                  |       | j                  j                  |d          | j                  j                  |d          | j                  j                  d|v r| j                   d   n|d          | j                  j                  | j                   d          | j                  j                  |d          | j                  j                  |d          | j                  j                  | j                         y )	NrJ   r   r   r   r   r   r   r   )r:   r?   r   r   r   r   r   r   r   r   r   r   r0   rU  s      r'   r   zGemmaModel.set_gguf_parameters
  s3   ,,12++G4M,NO--gm.DE((5009L1MN''0E(FG**RgkrRr4<<8M+Nx  AV  yW  	X//^0LM''
(;<))'**=>&&tzz2r&   c                    ~|dk(  rt         j                  d|d       g S |j                  d      r|dz   }| j                  |      |fgS Nr  zSkipping get tensor 1 in safetensors so that convert can end normally.norm.weightr   rf   r  r  r   r   s       r'   r   zGemmaModel.modify_tensors
  \     ##LL/x7hijI =='#aJ%%d+Z899r&   Nrf  )
r   r   r   rV   r  GEMMArE   r   r   r   r%   r&   r'   r  r  
  s     &&J5 3:r&   r  Gemma2ForCausalLMc                  L    e Zd Zej                  j
                  Zd Zd ZddZ	y)Gemma2Modelc                Z    | j                          | j                  j                  d       y r   )r  r?   rK  r   s    r'   r   zGemma2Model.set_vocab   s"    %%'--e4r&   c                   | j                   }|d   }| j                  j                  |d          | j                  j                  |d          | j                  j	                  |       | j                  j                  |d          | j                  j                  |d          | j                  j                  d|v r| j                   d   n|d          | j                  j                  | j                   d          | j                  j                  |d          | j                  j                  |d          | j                  j                  | j                         | j                  j                  | j                   d	          | j                  j                  | j                   d
          | j                  j                  | j                   d          y )NrJ   r   r   r   r   r   r   r   attn_logit_softcappingfinal_logit_softcappingrA  )r:   r?   r   r   r   r   r   r   r   r   r   r   r0   add_attn_logit_softcappingadd_final_logit_softcappingrB  rU  s      r'   r   zGemma2Model.set_gguf_parameters  s   ,,12++G4M,NO--gm.DE((5009L1MN''0E(FG**RgkrRr4<<8M+Nx  AV  yW  	X//^0LM''
(;<))'**=>&&tzz233LL12	
 	44LL23	
 	++DLL9I,JKr&   c                    ~|dk(  rt         j                  d|d       g S |j                  d      r|dz   }| j                  |      |fgS r  r  r   s       r'   r   zGemma2Model.modify_tensors  r  r&   Nrf  )
r   r   r   rV   r  GEMMA2rE   r   r   r   r%   r&   r'   r  r  
  s!    ''J5
L,:r&   r  Starcoder2ForCausalLMc                  8    e Zd Zej                  j
                  Zy)StarCoder2ModelN)r   r   r   rV   r  
STARCODER2rE   r%   r&   r'   r  r  +  s    ++Jr&   r  Rwkv6ForCausalLMc                  L    e Zd Zej                  j
                  Zd Zd ZddZ	y)
Rwkv6Modelc           	        | j                   dz  j                         sJ | j                  j                  dd      }dj	                  d      g}t
        j                  j                  g}t        | j                   dz  dd      5 }|j                         }|D ]  }|j                  d      }t        |      d	k\  sJ t        j                  dj                  |d
d             t        |d         }	}t!        |t"              r|j	                  d      n|}t!        |t$              sJ t        |      |	k(  sJ t'        |      dd }
|j)                  |
j	                  d             |j)                  t
        j                  j*                          	 d d d        |t        |      z
  }|dk\  sJ t-        t        |      |      D ]O  }|j)                  d| dj	                  d             |j)                  t
        j                  j.                         Q | j0                  j3                  d       | j0                  j5                  |       | j0                  j7                  |       t        j8                  | j                   d      }d|_        |j=                  dd       |j?                  | j0                         y # 1 sw Y   -xY w)Nzrwkv_vocab_v20230424.txtr  i   z<s>r   r   r   r  r   r   r>  r   r   r  r  rwkvFr  z
rwkv-worldr  i  ) r.   r   r:   r   r  rV   r  r!   r   	readlinesr  r[   astliteral_evalr9  r;   r   r   r  r  rt  r   r  r#   r?   r  r  r  r  r  r  r  )rl   r  r  r  r   lineslinerG  r  	token_lenr  	remainderr  r  s                 r'   r   zRwkv6Model.set_vocab4  sM   !;;DDFFF\\%%lE:
$||G45#~~556$..#==sWU 	7YZKKME 	7

35zQ&#&#3#3CHHU1R[4I#JCPUVXPYNy1;E31GW-U!%///5zY..."&u+a"3
j//89 5 56	7	7 V,	A~~s6{J/ 	3AMMD1+,,W56OODNN112	3 	,,V4''/((2))$..eL&2#((4!!$"2"233	7 	7s   D
J::Kc                   | j                   d   }| j                   d   }| j                   d   }| j                   d   }| j                   d   }| j                   d   | j                   d   nt        |dz  dz  dz        }|d	k(  rd
nd}|d	k(  rdnd
}| j                  j                  d       | j                  j	                  |       | j                  j                  |       | j                  j                  |       | j                  j                  |       | j                  j                  |       | j                  j                  |       | j                  j                  |       | j                  j                  |       | j                  j                  | j                         | j                  j                  d       y )NrJ   	head_sizer   r   rescale_everyr   g      @    rT  @   rZ     r   )r:   r;   r?   r   r   r   r   add_rescale_every_n_layersadd_wkv_head_sizeadd_time_mix_extra_dimadd_time_decay_extra_dimr   r   r0   r   )	rl   r<   r  r   r   rescale_every_n_layersr   time_mix_extra_dimtime_decay_extra_dims	            r'   r   zRwkv6Model.set_gguf_parametersV  s   ll#67LL-	ll=1&:;!%o!>AENaAbAnDLL)<=tw  zE  HK  zK  PR  yR  UW  yW  uX#.$#6RB&1T&9sr 	++G4--k:((5++N;334JK**95//0BC112FG001BC&&tzz2 	''*r&   c              #    K   | j                  |      }|j                  d      s|j                  d      s|dz  }|j                  d      s"|j                  d      s|j                  d      r|j                  dd      }|j                  d      r|j                  dd	d      }| j                  d
   }|dkD  rB|j                  d      s|j                  d      r |j                  d	t        ||z        z        }||f y w)Nr  re  ztime_mix_w1.weightztime_mix_decay_w1.weightztime_mix_decay_w2.weightr   r   ztime_mix_w2.weightr   r  ztime_mix_output.weightzchannel_mix_value.weight)r   r  r.  rP  r:   div_r;   )rl   r   r   r   r   r  s         r'   r   zRwkv6Model.modify_tensorso  s    ''-!!),0A0A'0J	!H12h6G6GHb6cgogxgx  zT  hU#--a3J12#++Aq!4J!%o!>!A%  !9:h>O>OPj>k'__Q#c=S6S2T-TU
$$s   C?DNrf  )
r   r   r   rV   r  RWKV6rE   r   r   r   r%   r&   r'   r  r  0  s!    &&J 4D+2%r&   r  MambaForCausalLMMambaLMHeadModelFalconMambaForCausalLMc                  P    e Zd Zej                  j
                  Zd Zd ZdZ	ddZ
y)
MambaModelc                h   | j                   d   }| j                   j                  dd      }|| z   |z  }|| j                   d<   | j                  dz  j                         r| j	                          y | j                  dz  j                         r| j                          y | j                  d|       y )Nr  pad_vocab_size_multiplerY  r  r  r=  )r:   r   r.   r   r   r  r^  )rl   r  	pad_vocabs      r'   r   zMambaModel.set_vocab  s    \\,/
LL$$%>B	 "iZ/09<
%/\"NN--668  "nn0099;))+ ##J
;r&   c                *   | j                  ddg      }| j                  ddgd      xs d}| j                  dd	gd      xs d
|z  }| j                  ddgd      xs d}| j                  ddgd      xs |dz   }| j                  ddgd      xs d}d}| j                  dgd      dv rd}|d
|z  k(  sJ | j                  j                  d       | j                  j                  |       | j                  j	                  d       | j                  j                  d       | j                  j                  | j                         | j                  j                  |       | j                  j                  |       | j                  j                  |       | j                  j                  |       | j                  j                  |       | j                  j                  |       | j                  j                  | j                         y )Nr   r  conv_kerneld_convTr   r   r   d_innerr   
state_sized_state   time_step_rankdt_rankir   r   r  FrJ  )falcon_mambar  r   )r]   r?   r   r   r   r   r   r<   add_ssm_conv_kerneladd_ssm_inner_sizeadd_ssm_state_sizeadd_ssm_time_step_rankr   add_ssm_dt_b_c_rmsr   r0   )rl   r  r  r  r  r  r   use_dt_b_c_norms           r'   r   zMambaModel.set_gguf_parameters  s   ""M#CD""M#Bd"SXWX""$7#Cd"SbWX[bWb""L#Cd"SYWY '')9y(IY]'^sdkordrbs'')=~(NY]'^fbf\NT:>OO"O!g+%%%++E2--g6003''*(()9)9:,,V4++G4++G4//8//=++O<&&tzz2r&   Nc                   ~| j                  t        j                  j                        }| j                  t        j                  j                        }| j                  |      }|j                  d      r.t        j                  d|z          t        j                  |       }| j                  B||k(  r=t        j                  | j                  |      r)t        j                  | d| d       g S ||k(  r|| _        ||fgS )Nz.A_logzA_log --> A ==> z is equivalent to z
, omitting)r   rV   r  r(  r'  r   r  rf   r  rd   exp	_tok_embdequal)rl   r   r   r   output_nametok_embd_namer   s          r'   r   zMambaModel.modify_tensors  s    --d.?.?.F.FG//0A0A0L0LM''-=="LL+h67))J//J >>%(k*A{{4>>:6},>}oZXY	&'DN:&''r&   rf  )r   r   r   rV   r  MAMBArE   r   r   r  r   r%   r&   r'   r  r    s'    &&J<"3< I(r&   r  CohereForCausalLMc                  T     e Zd Zej                  j
                  Z fdZ fdZ xZ	S )CommandR2Modelc                d    t        |   |i | | j                  ddg      | j                  d<   y )Nr  r   )r  rp   r]   r:   r  s      r'   rp   zCommandR2Model.__init__  s8    $)&)
 372B2BDVXqCr2s./r&   c                    t         |           | j                  j                  | j                  d          | j                  j                  t        j                  j                         y )Nr  )	r  r   r?   r  r:   r  rV   r  NONEr  s    r'   r   z"CommandR2Model.set_gguf_parameters  sK    #%((m)DE..t/C/C/H/HIr&   )
r   r   r   rV   r  	COMMAND_RrE   rp   r   r#  r$  s   @r'   r"  r"    s$    **JtJ Jr&   r"  OlmoForCausalLMOLMoForCausalLMc                  R     e Zd Zej                  j
                  Z fdZddZ xZ	S )	OlmoModelc                    t         |           | j                  j                  d       | j                  j                  d      }|| j                  j                  |       y y )Nr  r  )r  r   r?   r   r:   r   r  )rl   r  r  s     r'   r   zOlmoModel.set_gguf_parameters  sU    #%++D1<<##J/**84  r&   c                    ~| j                   d   }| j                   j                  d      }|j                  d      rt        j	                  |||      }|j                  d      rt        j	                  |||      }| j                  |      |fgS r  r  r  s         r'   r   zOlmoModel.modify_tensors  s    34LL$$%:;	==)#++JGJ==)#++J	JJ%%d+Z899r&   rf  )
r   r   r   rV   r  OLMOrE   r   r   r#  r$  s   @r'   r*  r*    s     %%J5:r&   r*  Olmo2ForCausalLMc                  8    e Zd Zej                  j
                  Zy)
Olmo2ModelN)r   r   r   rV   r  OLMO2rE   r%   r&   r'   r0  r0    s    &&Jr&   r0  OlmoeForCausalLMc                  l     e Zd ZU ej                  j
                  Z fdZdZde	d<   ddZ
 fdZ xZS )
OlmoeModelc                    t         |           | j                  j                  d       | j                  j                  d      x}| j                  j                  |       y y )Nr  r  )r  r   r?   r   r:   r   r   )rl   r   r  s     r'   r   zOlmoeModel.set_gguf_parameters  sV    #%//5))-88IE--i8 Fr&   Nr0  rQ  c           	        |j                  d      dk7  r| j                  d   }|J | j                  (t        | j                        D cg c]  }i  c}| _        || j                  |   |<   t        | j                  |         |dz  k\  rg }dD ]  }g }t        |      D ]?  }	d| d|	 d| d	}
|j                  | j                  |   |
          | j                  |   |
= A t        j                  |d
      }d| d| d	}| j                  |      }|j                  ||f        |S g S | j                  |      |fgS c c}w r  r  r  s                r'   r   zOlmoeModel.modify_tensors  r!  r  c                    t         |           | j                  M| j                  D cg c]  }|j                         D ]  }|  }}}t	        |      dkD  rt        d|       y y c c}}w r  r  r  s       r'   rB  zOlmoeModel.prepare_tensors4  r  r  rf  )r   r   r   rV   r  OLMOErE   r   rQ  rl  r   rB  r#  r$  s   @r'   r4  r4    s6    &&J9 04H,3":JD Dr&   r4  JinaBertModelJinaBertForMaskedLMc                  j     e Zd Zej                  j
                  Z fdZ fdZ fdZ	d fdZ
 xZS )JinaBertV2Modelc                L    t        |   |i | | j                  d   | _        y )Nr   )r  rp   r:   r   r  s      r'   rp   zJinaBertV2Model.__init__B  s'    $)&)!%.A!Br&   c              #  L  K   t         |          D ]  \  }}d|v r|d | j                  d d f   }|j                  dd      }|j                  dd      }|| j                  d d d f   }|j                  dd      }|j                  dd      }||f ||f ||f  y w)Ngated_layergated_layersgated_layers_wup_gated_layergated_layers_v)r  rb   r   r  )rl   r   r   d1name1d2name2r  s          r'   rb   zJinaBertV2Model.get_tensorsF  s     '-/ 	JD$$14111145^5EF&68HI$001145^5EF&68HIRiRi*	s   B!B$c                   d}t        | j                  dz  dd      5 }t        j                  |      d   }d d d        |dk(  rt        |           n@|dk(  r,| j                          | j                  j                  d       nt        d	| d
      | j                  j                  d       | j                  j                  d       y # 1 sw Y   xY w)NBertTokenizerr  r   r   r   tokenizer_classRobertaTokenizerr   z
Tokenizer z# is not supported for JinaBertModelT)r   r.   r   r   r  r   r   r?   r  r  rX  rZ  )rl   rJ  r   r  s      r'   r   zJinaBertV2Model.set_vocabU  s    )$..#::C'R 	>VW"iil+<=O	> o-G 22  "11!4%
?2CCf&ghh**40**40	> 	>s   CCc                R    |j                  d      r|dd  }t        | 	  |||      S )Nzbert.r   )rs  r  r   r  s       r'   r   zJinaBertV2Model.modify_tensorsd  s0     ??7#8Dw%j$<<r&   rf  )r   r   r   rV   r  JINA_BERT_V2rE   rp   rb   r   r   r#  r$  s   @r'   r<  r<  >  s,    --JC1= =r&   r<  OpenELMForCausalLMc                  ~     e Zd Zej                  j
                  Zedd       Z fdZ	d Z
d Zdd	 fdZd
dZ xZS )OpenELMModelc                b    t        |t        | |dz  z         |z  |z        }|d| z  k  r||z  }|S )Nr   g?)r  r;   )r  divisornew_vs      r'   _make_divisiblezOpenELMModel._make_divisibleq  sA     GSWq[1W<wFG37?WEr&   c                2   t        |   |i | | j                  d   }| j                  d   }| j                  d   | _        | j                  d   | _        | j                  d   | _        |D cg c]%  }t        j                  || j                  z  |      ' c}| _        t        | j                  t              rt        | j                  d   t              sJ t        | j
                  t              rt        | j
                  d   t              sJ y c c}w )Nffn_multipliersffn_dim_divisor	model_dimr  num_query_headsr   )r  rp   r:   _n_embd_num_kv_heads_num_query_headsrP  rT  	_ffn_dimsr   r  r;   )rl   r  r  rV  rW  
multiplierr  s         r'   rp   zOpenELMModel.__init__z  s    $)&)'+||4E'F#||,=> LL5(,^(D+/<<8I+J .%
 ((dll)BOT%
 $,,d3
4CUCUVWCXZ]8^^^$//6:dF[F[\]F^`c;ddd;d%
s   /*Dc                    	 | j                          y # t        $ r" | j                  d| j                  d          Y y w xY w)Nr?  r  )r  r#  r^  r:   r   s    r'   r   zOpenELMModel.set_vocab  s>    	M))+  	M##Kl1KL	Ms    (>>c                N   | j                   }| j                  d   }d}| j                  t        | j                        k(  sJ | j                  t        | j
                        k(  sJ | j                  t        | j                        k(  sJ | j                  j                  | j                         | j                  j                  | j                  d          | j                  j                  |       | j                  j                  | j                         | j                  j                  | j
                         | j                  j                  | j                         | j                  j                  | j                  d          | j                  j                  d       | j                  j!                  t#        ||z               | j                  j%                  |       | j                  j'                  |       | j                  j)                  | j*                         y )Nr   rf  max_context_lengthrope_freq_constantgư>)rZ  r:   r<   r[   r[  r\  r]  r?   r   r   r   r   r   r   r   r   ru  r;   r   r   r   r0   )rl   r   r   r4  s       r'   r   z OpenELMModel.set_gguf_parameters  s   <<
+3t'9'9#::::3t'<'<#====3t~~#6666(()9)9:++DLL9M,NO--f500@''(=(=>**4+=+=>++DLL9M,NO//511#g6H2IJ''1))(3&&tzz2r&   c                J    d|v r| j                   d   S t        | 	  ||      S )NrI   num_transformer_layers)r:   r  r]   )rl   r|   r}   r  s      r'   r]   zOpenELMModel.find_hparam  s-    << 899w"422r&   c              #  0  K   |{|d| dk(  rr| j                   |   }| j                  t        j                  j                  |      |d | f | j                  t        j                  j
                  |      ||d  f y | j                  |      |f y w)Nztransformer.layers.z.ffn.proj_1.weight)r]  r   rV   r  r   r!  r   )rl   r   r   r   r  s        r'   r   zOpenELMModel.modify_tensors  s      ?t)<SEAS'TT^^C(F**4+<+<+E+EsKZX_Y_M`aa**4+<+<+C+CSI:V\V]K^__##D):66s   BB)r  zfloat | intrR  r;   r  r;   r_  r`  rf  )r   r   r   rV   r  OPENELMrE   rn  rT  rp   r   r   r]   r   r#  r$  s   @r'   rP  rP  m  s?    ((J e M3,3	7r&   rP  ArcticForCausalLMc                  r     e Zd ZU ej                  j
                  Zd Z fdZdZ	de
d<   ddZ fdZ xZS )	ArcticModelc                   ddl m} | j                  dz  }|j                         s-t        j                  d|        t        j                  d        |       }|j                  t        |             | j                  j                  d|j                               }t        |      D cg c]  }d| dj                  d	       }}d
g|z  }t        j                   g|z  }t        |j                               D ]  }	|j#                  |	      }
|
j                  d	      }|j%                  |	      }t        j&                  }|j)                  |	      rt        j*                  }ne|j-                  |	      rt        j.                  }nC|j1                  |	      rt        j                   }n!|j3                  |	      rt        j4                  }|||	<   |||	<   |||	<    | j                  dz  }|j                         rt7        |dd	      5 }t9        j:                  |      }d|v r|d   }|j=                         D ]  \  }	}t?        |	      }	|	|k\  rt        jA                  d|	 d|dz
          5|d   }t        jB                  }d
}d|v r0|d   r+||d   k(  rt        j*                  }nt        j.                  }d}t        jE                  d|	 d| d| d|dd	       |j                  d	      ||	<   |||	<   |||	<    d d d        | jF                  jI                  d       | jF                  jK                  d       | jF                  jM                  |       | jF                  jO                  |       | jF                  jQ                  |       tS        jT                  | j                  tW        |            }|jY                  | jF                         y c c}w # 1 sw Y   xY w)Nr   r  r  r:  r   r  r  r  r   r  r  r   r   r  r  r  r  r  	unk_tokenr  zSetting added token z to 'z	' (type: z	, score: z.2frM   r	  r
  r  )-r"  r  r.   r   rf   rr  r@  rs  r$  r   r:   r   r  r  r  r   r#   r%  r&  r   r'  r    r(  r!   r)  r*  r$   r   r   r   r  r;   r  r"   rg   r?   r  r  r  r  r  rV   r  r[   r  )rl   r  r+  r  r  r  r  r  r  r,  r-  r.  r/  r0  r3  r   r4  r  
token_jsontoken_content
token_typetoken_scorer  s                          r'   r   zArcticModel.set_vocab  s    	9*;;%%'LL?>*:;<HHQK +,	s>23\\%%lI4H4H4JK
DI*DUVqaS{11':VV'j:56==>Ki2245 	)H''1E<<(D&&x0E-44G""8,199$$X.199##H-188!!(+166#F8$F8!(HX%	), !%1H H ((*+S7C 7q(,		!%)-BB+@AW+X(0D0J0J0L 7,*#&x=#z1"LL=
B\]gjk]k\l)mn$(29(=%<%I%I
&. &3I9N,0Ek0RR-D-L-L
-D-L-L
*-K&:8*E-Xablammv  xC  DG  wH  HI  %J  K+8+?+?+Hx(-7*+6x(-77: 	,,W5**95''/))&1((2))$..#f+N!!$"2"23A W87 7s   +OC2OOc                    t         |           | j                  }| j                  j	                  |d          | j                  j                  |d   |d   z         y )Nr  r   r   )r  r   r:   r?   rM  ru  rl   r:   r  s     r'   r   zArcticModel.set_gguf_parameters  sT    #%,,''(=>11'-2HGTiLj2jkr&   Nr0  rQ  c           	        | j                   d   }| j                   j                  d      }|j                  d      rt        j	                  |||      }|j                  d      rt        j	                  |||      }|j                  d      dk7  r| j                   d   }|J | j                  (t        | j                        D cg c]  }i  c}| _        || j                  |   |<   t        | j                  |         |dz  k\  rg }d	D ]  }	g }
t        |      D ]?  }d
| d| d|	 d}|
j                  | j                  |   |          | j                  |   |= A t        j                  |
d      }d| d|	 d}| j                  |      }|j                  ||f        |S g S | j                  |      |fgS c c}w )Nr   r   r  r  rU  r>  r   r   rV  r  rZ  r  r  r   rz  r[  r\  r]  r^  s                  r'   r   zArcticModel.modify_tensors  s   34LL$$%:;	==)#++JGJ==)#++J	JJ 99/0B6%89I?"?}}$-243C3C-D E E'1DMM#t$4==%&)a-746 . ;C*,E$Y/ 6"/u4NseSTUXTYY` aT]]3%7%>? MM#.u56
 "'U!:J$+C50Fse7"SK#33K@HNNHj#9:; 	%%d+Z8997 !Fr`  c                    t         |           | j                  M| j                  D cg c]  }|j                         D ]  }|  }}}t	        |      dkD  rt        d|       y y c c}}w r  r  r  s       r'   rB  zArcticModel.prepare_tensorsE  r  r  rf  )r   r   r   rV   r  ARCTICrE   r   r   rQ  rl  r   rB  r#  r$  s   @r'   ri  ri    s>    ''JR4hl 04H,3+:ZD Dr&   ri  DeepseekV2ForCausalLMc                  r     e Zd ZU ej                  j
                  Zd Z fdZdZ	de
d<   ddZ fdZ xZS )	DeepseekV2Modelc                $    | j                          y rw   r   r   s    r'   r   zDeepseekV2Model.set_vocabS  r   r&   c                   t         |           | j                  }| j                  j	                  |d          | j                  j                  |d          d|v r#|d   | j                  j                  |d          | j                  j                  |d          | j                  j                  |d   |d   z          | j                  j                  |d          | j                  j                  |d          | j                  j                  |d	          | j                  j                  |d
          | j                  j                  |d          | j                  j                  |d          | j                  j                  d      d| j                  d   v r| j                  d   j                  d      dk(  r| j                  j!                  t"        j$                  j&                         | j                  j)                  | j                  d   d          | j                  j+                  | j                  d   d          | j                  j-                  d|d   d   z         y y y y )Nfirst_k_dense_replacer  r  r  r  r  
v_head_dimr  n_routed_expertsn_shared_expertsrouted_scaling_factorr  r  rT   r  rk  rH  mscale_all_dim)r  r   r:   r?   add_leading_dense_block_countrM  r  r  r   r   r  r   add_expert_shared_countadd_expert_weights_scaleru  r   r  rV   r  r  r  r  add_rope_scaling_yarn_log_mulrq  s     r'   r   z#DeepseekV2Model.set_gguf_parametersV  s,   #%,,66w?V7WX''(=>G#(>(J,,W]-CD))'.*AB''0B(CgN`Fa(ab))',*?@77@W8XY))'2D*EF009K1LM11':Q2RS11':L2MN<<N+7HUcHd<d||N+//76A  66t7K7K7P7PQ  88n9UV^9_`  >>t||N?[\~?  A  >>sW^E\]mEn?no	 B =e7r&   Nr0  rQ  c           	        |j                  d      dk7  r| j                  d   }|J | j                  (t        | j                        D cg c]  }i  c}| _        || j                  |   |<   t        | j                  |         |dz  k\  rg }dD ]  }g }t        |      D ]?  }	d| d|	 d| d	}
|j                  | j                  |   |
          | j                  |   |
= A t        j                  |d
      }d| d| d	}| j                  |      }|j                  ||f        |S g S | j                  |      |fgS c c}w )Nzmlp.expertsr>  r|  r   r  r  r  r  r  r   rz  r  r  s                r'   r   zDeepseekV2Model.modify_tensorsp  sg   99]#r)%78I?"?}}$-243C3C-D E E'1DMM#t$4==%&)a-746 D ;F*,E$Y/ 6"/uM#axw WT]]3%7%>? MM#.u56
 "'U!:J$1#mF87"SK#33K@HNNHj#9:; 	%%d+Z8997 !Fr  c                    t         |           | j                  M| j                  D cg c]  }|j                         D ]  }|  }}}t	        |      dkD  rt        d|       y y c c}}w r  r  r  s       r'   rB  zDeepseekV2Model.prepare_tensors  r  r  rf  )r   r   r   rV   r  	DEEPSEEK2rE   r   r   rQ  rl  r   rB  r#  r$  s   @r'   rw  rw  O  s<    **Jp0 04H,3":HD Dr&   rw  T5WithLMHeadModelT5ForConditionalGenerationMT5ForConditionalGenerationUMT5ForConditionalGenerationc                  ^     e Zd Zej                  j
                  Z fdZd Zd Z	ddZ
 xZS )T5Modelc                2    t        |   |i | d| _        y r   r  rp   shared_token_embeddings_foundr  s      r'   rp   zT5Model.__init__      $)&)-2*r&   c           	     
   dt         j                  d<   ddlm} ddlm} | j
                  dz  }|j                         s| j
                  dz  }|j                         st        d|       |j                         }|j                  t        |d	      j                                |j                  j                  d
k(  r!|j                  dk(  sJ | j                         S |j                  j                  dk(  sJ |j                   j"                  }|j                   j$                  }|j                   j&                  } |       }|j)                  t+        |             | j,                  j/                  d|j1                               }	t3        |	      D 
cg c]  }
d|
 dj5                  d       }}
dg|	z  }t6        j8                  g|	z  }t3        |j1                               D ]  }|j;                  |      }|j5                  d      }|j=                  |      }t6        j>                  }|jA                  |      rt6        jB                  }ne|jE                  |      rt6        jF                  }nC|jI                  |      rt6        j8                  }n!|jK                  |      rt6        jL                  }|||<   |||<   |||<    | j
                  dz  }|j                         rt        |dd      5 }tO        jP                  |      }|D ]W  }||   }||	k\  rtR        jU                  d| d|	dz
          ,|j5                  d      ||<   d||<   t6        jV                  ||<   Y 	 d d d        |	tY        |      kD  r|	tY        |      z
  }tR        j[                  d| d| d       t3        d|dz         D ]R  }
|j]                  t_        d|
 dd             |j]                  d       |j]                  t6        j8                         T | j`                  jc                  d       | j`                  je                  d       | j`                  jg                  |       | j`                  ji                  |       | j`                  jk                  |       | j`                  jm                  |       | j`                  jo                  |       |r| j`                  jq                  |       ts        jt                  | j
                  tY        |            }|jw                  | j`                         | j`                  jy                  d       | j`                  j{                  d       y c c}
w # 1 sw Y   xY wNr  r  r   r  ro  r  zspiece.modelr  rq  r   r   r  r  r  r   r  r  r   r   r  r  r  r   r!  r  r
  r  FT>rq  r  r"  r  rp  r.   r   r#  rt  ru  r   rv  r  rJ  r   r  rw  rx  r  r  r$  r   r:   r   r  r  r  r   r#   r%  r&  r   r'  r    r(  r!   r)  r*  r$   r   r   rf   r5  r"   r[   r  rt  r  r?   r  r  r  r  r  rK  r  r  rV   r  r  rX  rZ  rl   r  rG   r+  ry  rz  r  r  r  r  r  r  r  r  r,  r-  r.  r/  r0  r1  r   r2  r~   r6  r  s                            r'   r   zT5Model.set_vocab      @H

;<8B*;; %%'!^^n<N%%'#&6~6F$GHH#..0++D,F,K,K,MN ++66!;!&&*;;;;0022&33>>!CCC(88II
0@@YY2BBWW*,	s>23\\%%lI4H4H4JK
DI*DUVqaS{11':VV'j:56==>Ki2245 	)H''1E<<(D&&x0E-44G""8,199$$X.199##H-188!!(+166#F8$F8!(HX#	)& !NN-@@$$&'w? 
N1$(IIaL!, NC05H:-xj@Z[ehi[iZj'kl '*zz'':F8$'.F8$)@)M)MHX&N
N F#"S[0ILL.yk9XYbXccdef1i!m, @ed1#QK'BCg& 7 > >?@
 	,,T2**95''/))&1((2--j9556HI556JK))$..#f+N!!$"2"23**51**40w W2
N 
N   U0A2U""U,c                   | j                  dgd      x}t        j                  d       d}| j                  j	                  |       | j                  j                  | j                  d          | j                  j                  | j                  d          | j                  j                  | j                  d          | j                  j                  | j                  d	          | j                  j                  | j                  d
          | j                  j                  | j                  d
          | j                  j                  | j                  d          | j                  j                  | j                  d          | j                  j                  | j                  d          | j                  j                  | j                  d          | j                  j!                  | j"                         y )Nr  Tr   JCouldn't find context length in config.json, assuming default value of 512   r  d_ffrL   r  d_kvr   relative_attention_num_bucketsdecoder_start_token_id)r]   rf   r5  r?   r   r   r:   r   r   r   r   r   r   add_relative_attn_buckets_countr   add_decoder_start_token_idr   r0   rl   r   s     r'   r   zT5Model.set_gguf_parameters
  s   %%}o%EEENNNghE++E2--dll9.EF00f1EF((l)CD''[(AB''V(<=))$,,v*>?++DLL9M,NO88Ff9gh//=Q0RS33DLLAY4Z[&&tzz2r&   c                    ~|dv r1| j                   s
d}d| _         nt        j                  d|d       g S | j                  |      |fgS N)zdecoder.embed_tokens.weightzencoder.embed_tokens.weightshared.weightr  TzSkipping shared tensor r  r  rf   r  r   r   s       r'   r   zT5Model.modify_tensors  [     bb55&5926th>opq	%%d+Z899r&   rf  )r   r   r   rV   r  T5rE   rp   r   r   r   r#  r$  s   @r'   r  r    s*    
 ##J3_1B3":r&   r  T5EncoderModelc                  ^     e Zd Zej                  j
                  Z fdZd Zd Z	ddZ
 xZS )r  c                2    t        |   |i | d| _        y r   r  r  s      r'   rp   zT5EncoderModel.__init__1  r  r&   c           	     
   dt         j                  d<   ddlm} ddlm} | j
                  dz  }|j                         s| j
                  dz  }|j                         st        d|       |j                         }|j                  t        |d	      j                                |j                  j                  d
k(  r!|j                  dk(  sJ | j                         S |j                  j                  dk(  sJ |j                   j"                  }|j                   j$                  }|j                   j&                  } |       }|j)                  t+        |             | j,                  j/                  d|j1                               }	t3        |	      D 
cg c]  }
d|
 dj5                  d       }}
dg|	z  }t6        j8                  g|	z  }t3        |j1                               D ]  }|j;                  |      }|j5                  d      }|j=                  |      }t6        j>                  }|jA                  |      rt6        jB                  }ne|jE                  |      rt6        jF                  }nC|jI                  |      rt6        j8                  }n!|jK                  |      rt6        jL                  }|||<   |||<   |||<    | j
                  dz  }|j                         rt        |dd      5 }tO        jP                  |      }|D ]W  }||   }||	k\  rtR        jU                  d| d|	dz
          ,|j5                  d      ||<   d||<   t6        jV                  ||<   Y 	 d d d        |	tY        |      kD  r|	tY        |      z
  }tR        j[                  d| d| d       t3        d|dz         D ]R  }
|j]                  t_        d|
 dd             |j]                  d       |j]                  t6        j8                         T | j`                  jc                  d       | j`                  je                  d       | j`                  jg                  |       | j`                  ji                  |       | j`                  jk                  |       | j`                  jm                  |       | j`                  jo                  |       |r| j`                  jq                  |       ts        jt                  | j
                  tY        |            }|jw                  | j`                         | j`                  jy                  d       | j`                  j{                  d       y c c}
w # 1 sw Y   xY wr  r  r  s                            r'   r   zT5EncoderModel.set_vocab5  r  r  c                   | j                  dgd      x}t        j                  d       d}| j                  j	                  |       | j                  j                  | j                  d          | j                  j                  | j                  d          | j                  j                  | j                  d          | j                  j                  | j                  d	          | j                  j                  | j                  d
          | j                  j                  | j                  d
          | j                  j                  | j                  d          | j                  j                  | j                  d          | j                  j                  | j                  d          | j                  j                  | j                          y )Nr  Tr   r  r  r  r  rL   r  r  r   r  )r]   rf   r5  r?   r   r   r:   r   r   r   r   r   r   r  r   r   r0   r  s     r'   r   z"T5EncoderModel.set_gguf_parameters  se   %%}o%EEENNNghE++E2--dll9.EF00f1EF((l)CD''[(AB''V(<=))$,,v*>?++DLL9M,NO88Ff9gh//=Q0RS&&tzz2r&   c                    ~|dv r1| j                   s
d}d| _         nt        j                  d|d       g S | j                  |      |fgS r  r  r   s       r'   r   zT5EncoderModel.modify_tensors  r  r&   rf  )r   r   r   rV   r  	T5ENCODERrE   rp   r   r   r   r#  r$  s   @r'   r  r  -  s(    **J3_1B3 :r&   JAISLMHeadModelc                  h     e Zd Zej                  j
                  Z fdZd Zd Z	ddZ
 fdZ xZS )	JaisModelc                
   t        |   |i | | j                  d   dk(  sJ | j                  d   dk(  sJ d| _        d| j                  v r| j                  d   | _        n%d| j                  v r| j                  d   | _        nJ d| _        d| j                  v r<d	| j                  v sJ | j                  d   | j                  d	   z  | _        d| _        y d
| j                  v r| j                  d
   | _        d| _        y J )Nr  r  position_embedding_typer  rf  mup_embeddings_scaleembeddings_scalemup_output_alphamup_width_scalewidth_scalerh  )r  rp   r:   r  r  max_alibi_biasr  s      r'   rp   zJaisModel.__init__  s   $)&) ||12h>>>||56'AAA !$!T\\1$(LL1G$HD!4<</$(LL1C$DD!5-$444#||,>?$,,O`BaaD " dll*#||M:D " 5r&   c                $    | j                          y rw   r   r   s    r'   r   zJaisModel.set_vocab  r   r&   c                .   | j                   j                  | j                  d          | j                   j                  | j                  d          | j                   j	                  | j                  d          | j                   j                  | j                  d          | j                   j                  | j                  d          | j                   j                  | j                  d          | j                   j                  | j                         y )NrK   r  r   r   r   r   r(  r   s    r'   r   zJaisModel.set_gguf_parameters  s    ((i)@A++DLL,GH--dll8.DE00i1HI''X(>?++DLL9M,NO&&tzz2r&   c                   ~g }|j                  d      r|S |j                  d      r}dt        j                  t        j                  | j                  d               z  }t        |d   j                               }t        t        j                  |      |z         | _        |S |j                  d      r|j                  dd      }| j                  |      }|| j                  t        j                  j                        k(  r"|j                  ||| j                   z  f       |S || j                  t        j                  j"                        k(  r"|j                  ||| j$                  z  f       |S |j                  ||f       |S )Nr*  zrelative_pe.slopesr   r   r   )r+  r,  r-  z.c_fc2.weightr   )r  ro  floorlog2r:   r  itemr  r  r.  r   r   rV   r  r'  rt  r  r(  r  )rl   r   r   r   rn  n_head_closest_log2	first_valr   s           r'   r   zJaisModel.modify_tensors  sZ   ,. ==,(N==.0
 #$tzz$))DLL<R2S'T"Tjm0023I#(9)=@S)S#T"TDN==^_#--a3J''-t..t/@/@/K/KLLNNHj43H3H&HIJ  001B1B1I1IJJNNHj43C3C&CDE  NNHj12r&   c                l    t         |           | j                  j                  | j                         y rw   )r  rB  r?   r  r  r  s    r'   rB  zJaisModel.prepare_tensors  s(    !++D,?,?@r&   rf  )r   r   r   rV   r  JAISrE   rp   r   r   r   rB  r#  r$  s   @r'   r  r    s3    %%J"83 DA Ar&   r  ChatGLMModelChatGLMForConditionalGenerationc                  v    e Zd Zej                  j
                  Zd Zed        Z	edd	d       Z
d Zd Zd
dZy)r  c                   | j                   }| j                  }g }g }g }ddlm} |j	                  |d      }|j                  dt        |j                                     }t        |j                         j                               |k  sJ g d}	g d|	z   }
t        |      D ]g  }|j                  |      }|dk(  rd}n|d	k(  rd
}n|dk(  rd}|j                  d      }d}t        |      dk7  rL||j                  j                  j                         k  r%|j                  j                  j!                  |      }||j                  j                  j                         k\  r||
v rt"        j$                  }nDt        |      dk(  r&d| dj                  d      }t"        j&                  }nt"        j(                  }|j+                  |       |j+                  |       |j+                  |       Nt"        j,                  }|j                  j                  j/                  |      rt"        j0                  }n|j                  j                  j3                  |      rt"        j$                  }nk|j                  j                  j5                  |      rt"        j&                  }n5|j                  j                  j7                  |      rt"        j8                  }|j+                  |       |j+                  |       |j+                  |       j | j:                  j=                  d       | j:                  j?                  d       | j:                  jA                  |       | j:                  jC                  |       | j:                  jE                  |       tG        jH                  | j                   t        |            }|jK                  | j:                         y )Nr   r  Tr  padded_vocab_size)z
<|system|><|user|>z<|assistant|>z<|observation|>)z[MASK]z[gMASK]z[sMASK]sopeopz<unk>r   z<bos>r   z<eos>r   r  r  r  r	  zchatglm-spmr  )&r.   r:   r  r  r  r   r[   r  r  r  r  _convert_id_to_tokenr  r  sp_modelr  	get_scorer   r!   r#   r"   rt  r   
is_unknownr    
is_control	is_unusedis_byter$   r?   r  r  r  r  r  rV   r  r  )rl   r.   r:   r  r  r  r  r  r  role_special_tokensr  r,  r-  r.  r/  r0  r  s                    r'   set_vocab_chatglm3zChatGLMModel.set_vocab_chatglm3  s   NN	,,   .!11)t1T	[[!4c):M:M:O6PQ
9&&(//12Z???\GJ]]j) )	%H228<E1}QQ<<(DE 5zQ8i.A.A.J.J.U.U.W#W!++44>>xH9..77BBDDN*5==GZ1_!(1-44W=D5<<G5BBGd#e$(-44G""++66x@199$$--88B199$$--77A188$$--55h?166MM$MM% OOG$S)	%V 	,,W5 	**=9''/))&1((2))$..#f+N!!$"2"23r&   c           	         ddl m}  |       }dj                  | j                  d      D cg c]  }|t	        |          c}      S c c}w r  r  r  s       r'   r  z"ChatGLMModel.token_bytes_to_stringR  r  r  Nc                >   |D cg c]  }t        |g       }}	 d }d }t        t        |d d |dd              D ].  \  }}| j                  |d   |d   z         }	|	#||	|k  s+|}|	}0 ||||k\  r	 |S |J |d | ||   ||dz      z   gz   ||dz   d  z   }c c}w r  r  r  s
             r'   r  zChatGLMModel.bpeX  r  r   c           	        d| j                   j                  dd      v r| j                          y | j                  }| j                   }g }g }ddlm} |j                  |d      }|d   }t        |j                         j                               |k  sJ | j                  |      }g }	i }
|j                  }|j                         D ]  \  }}||
t        j                  |      <   t        |      d	k(  r-t        j!                  |||
      }t        |      dk\  rt        |      dk  sJ |	j#                  dj%                  t'        t        j                  |                    |j)                         }i |
|j                         D ci c]  \  }}||
 }}}t+        |      D ]  }||vr?|j#                  d| d       |j#                  t,        j.                  j0                         G||   |v r|j#                  ||          |j2                  |   j4                  r*|j#                  t,        j.                  j6                         |j#                  t,        j.                  j8                         |j#                  ||          |j#                  t,        j.                  j:                          | j<                  j?                  d       | j<                  jA                  |       | j<                  jC                  |       | j<                  jE                  |       t-        jF                  |d      }|	|_$        |jK                  d|j)                         d          |jK                  d|j)                         d          |jK                  d|j)                         d          |jM                  | j<                         y c c}}w )NzTHUDM/chatglm3-6b_name_or_pathr  r   r  Tr  r  r   r  r      r  r  r  r  Fr  r  r  r  r  r  )'r:   r   r  r.   r  r  r  r  r  r  r  r  r  r  r  r[   r  rt  r9  r  r  r  rV   r  r#   r  r  r!   r"   r   r?   r  r  r  r  r  r   r  r  r  s                        r'   r   zChatGLMModel.set_vocabi  s5   $,,"2"2?B"GG##%NN	,, .!11)t1T	01
9&&(//12Z???((3#33*002 	UKE4?CE,44U;<5zQ!%%out%LFv;!#Fq(888MM#((3|'I'I6#RST	U  //1C[eC[{C[CaCaCcd/?{C{*ddz" 	7A%QCqk* 5 56q![0mA./11!4<<OODNN$:$:;OODNN$?$?@mA./ 5 56	7 	,,V4**62''/((2)))G%((	0I0I0KO0\]((	0I0I0KJ0WX((	0I0I0KO0\]!!$"2"239 es   Oc                \   | j                   j                  d| j                   j                  d            }| j                   j                  d| j                   j                  d            }| j                   j                  d|      }| j                  j                  | j                   j                  d|             | j                  j	                  |       | j                  j                  | j                   j                  dd|z               | j                  j                  | j                   d	          | j                  j                  |       | j                  j                  |       | j                  j                  | j                   d
          | j                  j                  | j                         | j                  j                  d       | j                  j                  d       d}d| j                   v r|| j                   d   z  }| j                  j                  |       y )Nr   ry  r   r   multi_query_group_numr  r  r   rL   layernorm_epsilonr  Fi'  
rope_ratio)r:   r   r?   r   r   r   r   r   r   r   r   r0   ru  rX  r   )rl   ry  r   r   	rope_freqs        r'   r   z ChatGLMModel.set_gguf_parameters  s   ,,""=$,,2B2B92MN!!(DLL,<,<=R,STLL$$%<fE	++DLL,<,<\7,ST--g6001A1ABSUVY`U`1ab((l)CD''/**95//=P0QR&&tzz211"5**51	4<<'!DLL$>>I++I6r&   c                t    ~|j                  d      rg S |j                  d      }| j                  |      |fgS )Nz.rotary_pos_emb.inv_freqztransformer.)r  removeprefixr   r   s       r'   r   zChatGLMModel.modify_tensors  s@    ==34I  0%%d+Z899r&   rw   r  rf  )r   r   r   rV   r  CHATGLMrE   r  rn  r  r  r   r   r   r%   r&   r'   r  r    sR    ((JA4F R R
   :4x7&:r&   NemotronForCausalLMc                  X     e Zd Zej                  j
                  Zd Z fdZddZ	 xZ
S )NemotronModelc                    | j                          | j                  j                  d       | j                  j                  d       y )Nr   r   )r  r?   rV  rT  r   s    r'   r   zNemotronModel.set_vocab  s6    %%'))!,))!,r&   c                   t         |           | j                  }| j                  j	                  |d          | j                  g d      }| j                  j                  |       | j                  g d      }| j                  ddg      }| j                  ddg      }| j                  j                  t        ||z        |z         d| j                  vs| j                  d   4| j                  j                  t        j                  j                         y | j                  j                  t        j                  j                         | j                  j                  | j                  d	          y )
Nr  )r   r   r   r.  )r,  r-  rope_percentr   r   r   r   r  r  )r  r   r:   r?   rM  r]   r   ru  r;   r  rV   r  r%  r  r  )rl   r:   r   r4  r   r   r  s         r'   r   z!NemotronModel.set_gguf_parameters  s/   #%,,''(=>%%&jk
++J7 ""#XY!!=(";<!!#8("CD11#g6F2G62QR -n1M1U2243G3G3L3LM2243G3G3N3NO44T\\(5KLr&   c                V    |j                  d      r|dz   }| j                  |      |fgS )Nr  r   )r  r   r   s       r'   r   zNemotronModel.modify_tensors  s2    
 =='#aJ%%d+Z899r&   rf  )r   r   r   rV   r  NEMOTRONrE   r   r   r   r#  r$  s   @r'   r  r    s"    ))J-
M*:r&   r  ExaoneForCausalLMc                  F    e Zd Zej                  j
                  Zd ZddZy)ExaoneModelc                x   | j                   }|d   dk(  sJ |d   }|d   }|d   }|j                  d|      }|d   }d|v r|d   nd	|z  }|d
   }| j                  j                  |       | j                  j	                  |       | j                  j                  |       | j                  j                  |       | j                  j                  |       | j                  j                  |       | j                  j                  |       | j                  j                  | j                         | j                   j                  d      x}	| j                  j                  |	       | j                  ddgd      }
|
|
nd}
| j                  j                  t        |
|d   |d   z  z               |j                  d      ud|d   v rm|d   j                  d      dk(  rU| j                  j!                  t"        j$                  j&                         | j                  j)                  |d   d          y y y y )Nr  silur   r   r   r   r   r   r   rL   r   r,  r-  Tr   rf  r  r  rT   r  )r:   r   r?   r   r   r   r   r   r   r   r   r0   r   r]   ru  r;   r  rV   r  r  r  )rl   r:   r   	embed_dimr  r  r   r   rL   r   r/  s              r'   r   zExaoneModel.set_gguf_parameters  s9   ,,-.&898")*C"DM*	12	{{#8)D !56<OSZ<ZG$78`adm`m\*

 	--i8''	2**<8++,CD//?001BC((4&&tzz2,,**<88JE//
;((*A:)NY](^)6)B11#mw}G]ahi~aG  7A  3B  	C;;~&2x7>CZ7Z~&**62h>  66t7K7K7R7RS  889PQY9Z[ ? 8[2r&   c              #    K   | j                  dgd      x}r|j                  dd      j                         dk(  r| j                  j                  dd      }| j                  j                  d	| j                  d
   | j                  d   z        }d|t	        j
                  d|dt        j                        |z  z  z  }|j                  dd      }|j                  dd      }|j                  dd      }| j                  j                  dd      }||z  }	||z  }
|	|
k7  sJ g }|D ]q  }dt        j                  z  |z  }||
k  r|j                  d       0||	kD  r|j                  |       G||z  |z
  ||z
  z  }|j                  dd|z
  |z  |z   z         s | j                  t        j                  j                        t	        j                  |t        j                        f y y y wrb  rl  rs  s                  r'   r   z"ExaoneModel.generate_extra_tensors  r}  r~  Nri  )	r   r   r   rV   r  EXAONErE   r   r   r%   r&   r'   r  r    s    ''J!\Fr&   r  GraniteForCausalLMc                  N     e Zd ZdZej
                  j                  Z fdZ xZ	S )GraniteModelz'Conversion for IBM's GraniteForCausalLMc                   | j                   j                  dd      x}rt        j                  d|       t        |           | j                   j                  d      x}r1| j                  j                  |       t        j                  d|       | j                   j                  d      x}r1| j                  j                  |       t        j                  d|       | j                   j                  d      x}r1| j                  j                  |       t        j                  d	|       | j                   j                  d
      x}r2| j                  j                  |       t        j                  d|       yy)a  Granite uses standard llama parameters with the following differences:

        - No head_dim support
        - New multiplier params:
            - attention_scale
            - embedding_scale
            - residual_scale
        - logits_scaling
        r   Nz.Ignoring head_dim (%s) from config for Graniteattention_multiplierz$gguf: (granite) attention_scale = %sembedding_multiplierz$gguf: (granite) embedding_scale = %sresidual_multiplierz#gguf: (granite) residual_scale = %slogits_scalingz!gguf: (granite) logits_scale = %s)r:   poprf   r5  r  r   r   r?   add_attention_scalerg   r  r  r  )rl   r   attention_scaler  r  logits_scaler  s         r'   r   z GraniteModel.set_gguf_parameters5  s:    ||''
D9989NNKXV#% #ll../EFF?F00AKK>P"ll../EFF?F00AKK>P!\\--.CDD>D//?KK=~N<<++,<==<=,,\:KK;\J >r&   )
r   r   r   __doc__rV   r  GRANITErE   r   r#  r$  s   @r'   r  r  0  s!    1((JK Kr&   r  GraniteMoeForCausalLMc                  P     e Zd ZdZej
                  j                  Zd fdZ xZ	S )GraniteMoeModelz*Conversion for IBM's GraniteMoeForCausalLMc                ~   |j                  d      r| j                  d   }|j                  d   d|z  k(  sJ d       |dd|ddf   |d|dddf   }}| j                  t        j
                  j                  |      |f| j                  t        j
                  j                  |      |fgS t        | %  |||      S )a/  In modeling_granitemoe, the JetMoe implementation of parallel experts
        is used. This essentially merges w1 and w3 into a single tensor with 2x
        the hidden size that is then split during forward. To keep compatibility
        with existing mixtral support, we pull them apart here.
        z$block_sparse_moe.input_linear.weightr   r   r   z4Merged FFN tensor size must be 2 * intermediate_size.N)
r  r:   r  r   rV   r  FFN_GATE_EXP
FFN_UP_EXPr  r   )rl   r   r   r   ffn_dimgateupr  s          r'   r   zGraniteMoeModel.modify_tensorsW  s     ==?@ll#67G##B'1w;6n8nn6!#xx"23ZWXq@P5Q"D(():):)G)GMtT(():):)E)EsKRP 
 w%j$<<r&   rf  )
r   r   r   r  rV   r  GRANITE_MOErE   r   r#  r$  s   @r'   r  r  R  s    4,,J= =r&   r  !ChameleonForConditionalGenerationChameleonForCausalLMc                  h     e Zd Zej                  j
                  Z fdZd ZddZ	e
d        Z xZS )ChameleonModelc                    t         |           | j                  j                  | j                  j                  dd             y )N	swin_normF)r  r   r?   add_swin_normr:   r   r  s    r'   r   z"ChameleonModel.set_gguf_parameterso  s3    #%&&t||'7'7U'KLr&   c                $    | j                          y rw   r   r   s    r'   r   zChameleonModel.set_vocabs  r   r&   c                   |j                  d      rg S | j                  d   }| j                  j                  d      }| j                  j                  d      }|j                  d      rt        j                  |||      }|j                  d      rt        j                  |||      }|j                  d      rt        j                  |||      }|j                  d      rt        j                  |||      }| j                  |      |fgS )	Nzmodel.vqmodelr   r   r   rS  rT  )zq_norm.weightzq_norm.bias)zk_norm.weightzk_norm.bias)	rs  r:   r   r  rH  rP  r  r  r   )rl   r   r   r   r   r  r  s          r'   r   zChameleonModel.modify_tensorsv  s     ???+I34LL$$%:;	\\%%m4
==9:#++JGJ==9:#++J	JJ==9:';;JPZ[J==9:';;J	S]^J%%d+Z899r&   c                    ||z  }| d   j                  d|dz        j                         j                  dd      } | j                  |d      } | S )Nr   r   r   r>  )r  tr  repeat_interleave)r   r  r  r   s       r'   r  z"ChameleonModel._reverse_hf_permute  sT    (]''8q=9;;=EEaL
11'1=
r&   rf  )r   r   r   rV   r  	CHAMELEONrE   r   r   r   rn  r  r#  r$  s   @r'   r  r  j  s8     **JM:,  r&   r  c                     e Zd ZU ej                  Zded<   ded<   ej                  ej                  ej                  ej                  iZ
ded<   ej                  ej                  ej                  ej                  ej                  ej                  ej                  ej                   ej"                  ej$                  ej&                  ej(                  dZded	<   dd
Zedd       Zedd       Zedd       Zy)r   torch.dtyperc   z
torch.Sizer  zdict[torch.dtype, type]
_dtype_map)F64r  r-  r+  I64I32I16U8I8BOOLF8_E4M3F8_E5M2zdict[str, torch.dtype]_dtype_str_mapc                    | j                   | j                     }t        j                  t        j                  j	                  || j
                        | fd       S )Nc                "    | j                         S rw   )r  r  s    r'   <lambda>z'LazyTorchTensor.numpy.<locals>.<lambda>  s    AGGI r&   metar  r~  )r  rc   rV   LazyNumpyTensormeta_with_dtype_and_shaper  )rl   rc   s     r'   r  zLazyTorchTensor.numpy  sI    

+##%%??tzzR%
 	
r&   c                2    t        j                  ||d      S )Nr(  )r\  rc   r   )rd   empty)rs   rc   r  s      r'   r*  z)LazyTorchTensor.meta_with_dtype_and_shape  s    {{U6BBr&   c                    | j                   |j                            }t        |j                               } | | j	                  ||      |fd       }t        t        j                  |      S )Nc                    | d d  S rw   r%   r%  s    r'   r&  z8LazyTorchTensor.from_safetensors_slice.<locals>.<lambda>  s    fghifj r&   r'  )r"  	get_dtypetuple	get_shaper*  r   rd   r   )rs   st_slicerc   r  r6   s        r'   r   z&LazyTorchTensor.from_safetensors_slice  s]    ""8#5#5#78!&x'9'9';!<55eUC8+\jkELL$''r&   Nc                    ~|i }|t         j                  j                  u r|d   j                         S  | j                  |      |i |S )Nr   )rd   r   r  _wrap_fn)rs   r~  typesr  r  s        r'   __torch_function__z"LazyTorchTensor.__torch_function__  sL    >F5<<%%%7==?"!s||D!42622r&   )r  zgguf.LazyNumpyTensor)rc   r  r  ztuple[int, ...]r  r   )r2  r   r  r   )r%   N)r   r   r   rd   r   _tensor_typerl  re   r6  r  r  float64bfloat16int64int32int16r7  int8r2   float8_e4m3fnfloat8_e5m2r"  r  rm  r*  r   r6  r%   r&   r'   r   r     s    <<L 	rzzrzz+J'  }}}}}}{{{{{{kkjj

&&$$.N* $
 C C ( ( 	3 	3r&   r   c                    t        j                  d      } | j                  ddd       | j                  dt        d	       | j                  d
t        g ddd       | j                  ddd       | j                  dt        d	       | j                  ddd       | j                  ddd       | j                  dt        d d       | j                  ddd       | j                  dt
        dd       | j                  dt        d d!       | j                  d"dd#       | j                  d$dd%       | j                  d&t        d'	       | j                         S )(Nz5Convert a huggingface model to a GGML compatible file)descriptionz--vocab-only
store_truezextract only the vocab)actionhelpz	--outfilezSpath to write to; default: based on input. {ftype} will be replaced by the outtype.)rT   rD  z	--outtypef32f16bf16q8_0tq1_0tq2_0autorG  zoutput format - use f32 for float32, f16 for float16, bf16 for bfloat16, q8_0 for Q8_0, tq1_0 or tq2_0 for ternary, and auto for the highest-fidelity 16-bit float type depending on the first loaded tensor type)rT   choicesr
  rD  z--bigendianz'model is executed on big endian machinerG   zdirectory containing model filez--use-temp-filez^use the tempfile library while processing (helpful when running out of memory, process killed)z	--no-lazyz\use more RAM by computing all outputs before writing (use in case lazy evaluation is broken)z--model-namezname of the model)rT   r
  rD  z	--verbosezincrease output verbosityz--split-max-tensorsr   zmax tensors in each splitz--split-max-size0zmax size per split N(M|G)z	--dry-runzConly print out a split plan and exit, without writing any new filesz--no-tensor-first-splitz;do not add tensors to the first split (disabled by default)z
--metadataz9Specify the path for an authorship metadata override file)argparseArgumentParseradd_argumentr   r   r;   
parse_args)parsers    r'   rR  rR    s   $$KMF
|%   $b   #'_in a   l6   d.   ,m   Lk   S$    L(   C(   c(   LR   !,J   4H  
 r&   c                h   | j                  d      rt        | d d       dz  }nz| j                  d      rt        | d d       dz  dz  }nT| j                  d      rt        | d d       dz  dz  dz  }n+| j                         rt        |       }nt        d|  d      |dk  rt        d|  d	      |S )
NKr>  i  MGzInvalid split size: z5, must be a number, optionally followed by K, M, or Gr   z, must be positive)r  r;   	isnumericr   )	split_strr   s     r'   split_str_to_n_bytesrZ    s    #	#2$&			C	 	#2$&-			C	 	#2$&-4					N/	{:opqq1u/	{:LMNNHr&   c                    t               } | j                  r%t        j                  t        j                         n$t        j                  t        j
                         | j                  }|j                         s8t        j                  d| j                   d       t        j                  d       t        j                  j                  t        j                  j                  t        j                  j                   t        j                  j"                  t        j                  j$                  t        j                  j&                  t        j                  j(                  d}| j*                  dkD  xs | j,                  dk7  }| j.                  r,|r*t        j                  d       t        j                  d       | j0                  | j0                  }n|}t        j3                  d	|j4                          t6        j9                  |      }t;        j<                         5  || j>                     }|d
   d   }	 t6        jA                  |      } |||| jD                  | j.                  | jF                  | jH                  | jJ                  | j*                  tM        | j,                        | jN                  | jP                        }	| jR                  rHt        j3                  d       |	jU                          t        j3                  d|	jV                          nst        j3                  d       |	jY                          |r(|	jV                  jZ                   t\        j^                   n|	jV                  }
t        j3                  d|
        d d d        y # tB        $ r2 t        j                  d| d       t        j                  d       Y w xY w# 1 sw Y   y xY w)N)levelzError: z is not a directoryr   rE  r   rN  z*Error: Cannot use temp file when splittingzLoading model: architectureszModel z is not supported)r.   r0   r1   r3   r5   rm   rC   rA   rP   rQ   rR   rS   zExporting model vocab...z%Model vocab successfully exported to zExporting model...zModel successfully exported to )0rR  verboseloggingbasicConfigDEBUGINFOrG   rS  rf   rr  r@  rs  rV   r_   r,  rh   ri   r.  r)  r*  r`   rP   rQ   r5   outfilerg   r   r,   r\   rd   inference_modeouttyper  r  	bigendianno_lazyrQ  rA   rZ  rR   no_tensor_first_splitr]  ro  r1   rl  rX  rq  sep)r  r.   	ftype_mapis_splitr1   r:   rM  model_architecturemodel_classmodel_instanceout_paths              r'   mainrp  *  s   <D||'--0',,/

Iwtzzl*=>? !!))!!,,""..""..##00##00""**0I %%)GT-@-@C-GHhAB||LL		
KK/)..!123  +G				 F-$_5a8	778JKK
 %yW`37>>QUQcQc+/<<7;}}QUQ`Q`7;7M7M4HI\I\4]gkgsgs7;7Q7QS ??KK23&&(KK?@X@X?YZ[KK,-  "GO.2299:266(CUcUmUmHKK9(DE5F F # 	LL6"4!55FGHHHQK	F Fs1   $O=ND>O7OOOOO __main__)r  zargparse.Namespace)rY  r   r  r;   rj  )g
__future__r   r  r_  rO  r   r   rq  r~  r@  enumr   pathlibr   hashlibr   typingr   r   r	   r
   r   r   r   r   r   r   	itertoolsr   ro  r  r6  rd   r   r  rN   insertr   __file__rX  rV   	getLoggerrf   r   r(   r,   r  rq  r  r  r  r  r  r  r	  r  r)  rH  r  r  r  r  r  r  r  r  r&  r2  r8  rQ  re  rm  r  r  r  r  r  r  r  r  r"  r*  r0  r4  r<  rP  ri  rw  r  r  r  r  r  r  r  r  r  LazyBaser   rR  rZ  rp  r   r%   r&   r'   <module>r|     s:	   # 
     	 	 
    u u u    "**$HHOOAs4>009<=> 			<	(
g  :]3^L ^LB $%55 5 &5p "L1< < 2<~  !((u (( "((V "#J J $JD %'<=L7E L7 >L7^ #$g
% g
 %g
T #_52:% 2: 62:j '(3U 3 )3  &'<% < (<~ %'ACbcV@E V@ dV@r "$68LNbcOD OD dODd #$%%% %% %%%P !"1: 1: #1:h !"A A #AH $%3:5 3: &3:l %&1
E 1
 '1
h !"&3 &3 #&3R "#A A $A$ %&;DE ;D ';D| !"    # F  !2 2 "2. !"_@E _@ #_@D "#,( ,( $,(^ &' U   ( F &'i>U i> (i>X -~>E: E: ?E:P  !MY M "M: !#HIk=i k= Jk=\ "#/: /: $/:d #$+:% +: %+:\ '(,e , ), "#O% O% $O%d "$68PQH( H( RH(V #$JU J %J" !"!": : # #:4 "#' ' $' "#8D 8D $8Dv !67+=i += 8+=\ $%G75 G7 &G7T #$SD% SD %SDl '(KDe KD )KD\ #$,--../H:e H: 0 / . %H:V  !G:U G: "G:T !"OA OA #OAd  ABs:5 s: Cs:l %&%:E %: '%:P #$A% A %AH $%K: K &KB '(=l = )=. 34&'%U % ( 5%X>3dmm >3B<~$@FF zF r&   