sg`dZddlmZddlmZddlmZmZmZm Z m Z m Z dZ GddZ dZd Zd Zd Zd Zd Zd Z ddlZddlZddlmZm Z dZej0dZej0dZddddZdZ dGdZGddZedddZedgedezgedezgedd d!ged"d ged#gd$ed%d&d'ged(d'ged)d*j?d+d,Dzd-zged.ged/d0ged1gd2 Z ed3ged4ged5d6d7ged8d6ged9d7ged:d;gd<Z!d=Z"d>Z#e$dd?ge jKe!jKZ&e$d@Z'e$dAZ(GdBdCZ)dHdDe*dEe ee*fdFZ+y)IzPython implementation of llama grammar parser directly translated from C++ source file in vendor/llama.cpp/common/grammar-parser.cpp.)Path)groupby)AnySetListOptionalTupleUnionrootceZdZdefdZed dededdfdZed deee fdeddfdZ ed d ededdfd Z y ) LlamaGrammar_grammarc(||_t|_yN)rLLAMA_GRAMMAR_DEFAULT_ROOT_root)selfrargskwargss J/var/www/html/venv/lib/python3.12/site-packages/llama_cpp/llama_grammar.py__init__zLlamaGrammar.__init__s  / grammarverbosereturnc||S)N)r)clsrrs r from_stringzLlamaGrammar.from_strings G$$rfilec< t|5}|j}dddr|j ||St |jjd#1swY@xYw#t$r)}t|jjd|d}~wwxYw)Nz: error reading grammar file: rz5: error parsing grammar file: params_grammer is empty)openread Exception from_file__name__r ValueError)rr rfrerrs rr&zLlamaGrammar.from_files d #q&&( # ??7G?< <}}%%&&[ \   # # ==))**HN  s- A)AA)A&"A)) B2$BB json_schemac:|jt||S)Nr")rjson_schema_to_gbnf)rr+rs rfrom_json_schemazLlamaGrammar.from_json_schema-s2;?QQrN)T) r' __module__ __qualname__strr classmethodboolrr rr&r.rrrr r s00%#%%%% U39-    R3RRRRrr z root ::= (expr "=" ws term "\n")+ expr ::= term ([-+*/] term)* term ::= ident | num | "(" ws expr ")" ws ident ::= [a-z] [a-z0-9_]* ws num ::= [0-9]+ ws ws ::= [ \t\n]* ag root ::= (declaration)* declaration ::= dataType identifier "(" parameter? ")" "{" statement* "}" dataType ::= "int" ws | "float" ws | "char" ws identifier ::= [a-zA-Z_] [a-zA-Z_0-9]* parameter ::= dataType identifier statement ::= ( dataType identifier ws "=" ws expression ";" ) | ( identifier ws "=" ws expression ";" ) | ( identifier ws "(" argList? ")" ";" ) | ( "return" ws expression ";" ) | ( "while" "(" condition ")" "{" statement* "}" ) | ( "for" "(" forInit ";" ws condition ";" ws forUpdate ")" "{" statement* "}" ) | ( "if" "(" condition ")" "{" statement* "}" ("else" "{" statement* "}")? ) | ( singleLineComment ) | ( multiLineComment ) forInit ::= dataType identifier ws "=" ws expression | identifier ws "=" ws expression forUpdate ::= identifier ws "=" ws expression condition ::= expression relationOperator expression relationOperator ::= ("<=" | "<" | "==" | "!=" | ">=" | ">") expression ::= term (("+" | "-") term)* term ::= factor(("*" | "/") factor)* factor ::= identifier | number | unaryTerm | funcCall | parenExpression unaryTerm ::= "-" factor funcCall ::= identifier "(" argList? ")" parenExpression ::= "(" ws expression ws ")" argList ::= expression ("," ws expression)* number ::= [0-9]+ singleLineComment ::= "//" [^\n]* "\n" multiLineComment ::= "/*" ( [^*] | ("*" [^/]) )* "*/" ws ::= ([ \t\n]+) aT root ::= object value ::= object | array | string | number | ("true" | "false" | "null") ws object ::= "{" ws ( string ":" ws value ("," ws string ":" ws value)* )? "}" ws array ::= "[" ws ( value ("," ws value)* )? "]" ws string ::= "\"" ( [^"\\] | "\\" (["\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F]) # escapes )* "\"" ws number ::= ("-"? ([0-9] | [1-9] [0-9]*)) ("." [0-9]+)? ([eE] [-+]? [0-9]+)? ws # Optional space: by convention, applied in this grammar after literal chars when allowed ws ::= ([ \t\n] ws)? a$ # This is the same as json.gbnf but we restrict whitespaces at the end of the root array # Useful for generating JSON arrays root ::= arr value ::= object | array | string | number | ("true" | "false" | "null") ws arr ::= "[\n" ws ( value (",\n" ws value)* )? "]" object ::= "{" ws ( string ":" ws value ("," ws string ":" ws value)* )? "}" ws array ::= "[" ws ( value ("," ws value)* )? "]" ws string ::= "\"" ( [^"\\\x7F\x00-\x1F] | "\\" (["\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F]) # escapes )* "\"" ws number ::= ("-"? ([0-9] | [1-9] [0-9]*)) ("." [0-9]+)? ([eE] [-+]? [0-9]+)? ws # Optional space: by convention, applied in this grammar after literal chars when allowed ws ::= ([ \t\n] ws)? aZ root ::= object value ::= object | array | string | number | ("true" | "false" | "null") ws object ::= "{" ws ( string ":" ws value ("," ws string ":" ws value)* )? "}" ws array ::= "[" ws ( value ("," ws value)* )? "]" ws string ::= "\"" ( [^"\\\x7F\x00-\x1F] | "\\" (["\\bfnrt] | "u" [0-9a-fA-F]{4}) # escapes )* "\"" ws number ::= ("-"? ([0-9] | [1-9] [0-9]{0,15})) ("." [0-9]+)? ([eE] [-+]? [0-9] [1-9]{0,15})? ws # Optional space: by convention, applied in this grammar after literal chars when allowed ws ::= | " " | "\n" [ \t]{0,20} zn root ::= item+ # Excludes various line break characters item ::= "- " [^\r\n\x0b\x0c\x85\u2028\u2029]+ "\n" N)rrz" "?z[^a-zA-Z0-9-]+z[\r\n"]z\rz\n\")  "cbs|dk(r |dk(rdS|dk(r|dSd}|dkDr0|rddd|zzdz}nrddndjg|z}dfd |dkDr ||k7r|dz }||||z |dkD z }|Sd rdzndd }|dk(r r d d|d }|S||dz }|S)Nr?+r7 c|r rdn}|dk(ry|dk(rd|dSr|sd|d|dz ddSd|d|zjd|zzS) z - n=4, no sep: '(a (a (a (a)?)?)?)?' - n=4, sep=',', prefix: '("," a ("," a ("," a ("," a)?)?)?)?' - n=4, sep=',', no prefix: '(a ("," a ("," a ("," a)?)?)?)?' r>rr<r9()?Tprefix_with_sep)rstrip)up_to_nrCcontent item_ruleopt_repetitionsseparator_rules rrHz*_build_repetition..opt_repetitionss>a { +  a< \wir? " Owiq1d!S TTVW WyNW,446$.I IrrBr@)z*)?*)F)join)rG min_items max_itemsrIitem_rule_is_literalresult item_operatorrHs` ` @r_build_repetitionrRs1  >i1n[? " !^ 1[? " F1} N$:IaOi783>F/=.)+3LL i'FJ*1}i/# /)i"7UVWW MNNS0KI;VWX >n 1]O37F M q) )F MrceZdZddedefdZy) BuiltinRuleNrFdepsc(||_|xsg|_yr)rFrU)rrFrUs rrzBuiltinRule.__init__7s JB rr)r'r/r0r1listrrrrrTrT6s4rrTz[0-9]z("true" | "false") spacez[0-9] z[0-9] | [1-9] zJ("-"? integral-part) ("." decimal-part)? ([eE] [-+]? integral-part)? space integral-part decimal-partz("-"? integral-part) spacez1object | array | string | number | boolean | null)objectarraystringnumberbooleannullzS"{" space ( string ":" space value ("," space string ":" space value)* )? "}" spacer]valuez1"[" space ( value ("," space value)* )? "]" space"\"" z "-" c#&K|] }d|z yw)z [0-9a-fA-F]Nr).0ns r rfTsCQ}q(C)riri "\"" spacezQ[^"\\] | "\\" (["\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F])z"\"" char* "\"" spacecharz "null" space) r_rZrYr^integerrar[r\uuidrlr]r`za[0-9] [0-9] [0-9] [0-9] "-" ( "0" [1-9] | "1" [0-2] ) "-" ( "0" [1-9] | [1-2] [0-9] | "3" [0-1] )z([01] [0-9] | "2" [0-3]) ":" [0-5] [0-9] ":" [0-5] [0-9] ( "." [0-9] [0-9] [0-9] )? ( "Z" | ( "+" | "-" ) ( [01] [0-9] | "2" [0-3] ) ":" [0-5] [0-9] )z date "T" timedatetimez"\"" date "\"" spacez"\"" time "\"" spacez"\"" date-time "\"" space date-time)rorprqz date-stringz time-stringzdate-time-stringz[\U00000000-\U0010FFFF]z [^\x0A\x0D]dotz |.()[]{}*+?z []()|{}*+?c eZdZdZdZ ddededefdZdZde d efd Z d Z d Z d Z dZdZdedefdZdeeeefdeededeeeffdZdZy)SchemaConverterc||_||_||_||_dti|_i|_t|_y)Nspace) _prop_order _allow_fetch_dotall _raw_pattern SPACE_RULE_rules_refsset_refs_being_resolved)r prop_order allow_fetchdotall raw_patterns rrzSchemaConverter.__init__}sB%' ' Z   $'E!rc<tjd|}d|dS)NcJtj|jdSNr)GRAMMAR_LITERAL_ESCAPESgetgroup)ms rz1SchemaConverter._format_literal..s-11!''!*=rr7)GRAMMAR_LITERAL_ESCAPE_REsub)rliteralescapeds r_format_literalzSchemaConverter._format_literals'+// =w 7)1~rrrrctdkDsJddtffd djdgddS)zp not_literal('a') -> '[^a]' not_literal('abc') -> '([^a] | "a" ([^b] | "b" ([^c])?)?)?' rzEmpty literal not supportedic3K|}r|dk(rd|ddd|dnd|d|tdz kr3dj|d |dzEd{d yy7 w) N_z[^z\\] | z"\\"? "r7]r9 (rA)lenr)rcrmaybe_escaped_underscoresrecursers rrz,SchemaConverter.not_literal..recurses A(Q#X1#Um# !!A&&1#Qi3w)) $*sA$A4'A2( A4r<r@rJ)rintrL)rrrrrs`` `@r not_literalzSchemaConverter.not_literalsP7|a>!>> s  ww.gaj.#.//rcRtjd|}||jvs|j||k(r|}n^d}|||jvrD|j|||k7r.|dz }|||jvr|j|||k7r.||}||j|<|S)N-rr9)INVALID_RULE_CHARS_RErr|)rnameruleesc_namekeyrs r _add_rulezSchemaConverter._add_rules(,,S$7 4;; &$++h*?4*GCA*QC DKK/KK8*QC 01T9Q*QC DKK/KK8*QC 01T9Jqc"C C rschemaurlc8dtffd S)z Resolves all $ref fields in the given schema, fetching any remote schemas, replacing $ref with absolute reference URL and populating self._refs with the respective referenced (sub)schema dictionaries. recjt|tr|Dcgc] } | c}St|tru|jd}|D| jvr5|j dr j sJdddl}|jd}|d} jj|}|> j|j|j|}| j|<t|dk(s|ddk(r/|S|j d r } |}||d<ntd ||jddjd ddD]}|||vsJd |d |d|||}| j|<|S|jD] } | |Scc}w)N$refzhttps://zDFetching remote schemas is not allowed (use --allow-fetch for force)r#r9r=r<z#/zUnsupported ref /zError resolving ref z: z not in ) isinstancerWdictrr} startswithrxrequestssplit resolve_refsjsonrr(values) rexrefr frag_splitbase_urltargetselvrrrvisits rrz+SchemaConverter.resolve_refs..visits!T"*+,Qa,,At$eeFm?s$**'<~~j1 --bab-'%(YYs^ #-a=!%!9!>%)%6%6 ( S 1 6 6 8(&F4:DJJx0z?a/:b>R3G#)M-!'!$cUm$'& (+;C5)ABB"yy~b177.s< !Az :$t/PQRPS'TU s),)rL enumerate)rr alt_schemass`` r_generate_union_rulez$SchemaConverter._generate_union_rules&zz %.{%;   rc jdrjdsJdddidtdttt fdtfd dttt fffd j jrSd zd zS) a Transforms a regular expression pattern into a GBNF rule. Input: https://json-schema.org/understanding-json-schema/reference/regular_expressions Output: https://github.com/ggerganov/llama.cpp/blob/master/grammars/README.md Unsupported features: negative/positive lookaheads, greedy/non-greedy modifiers. Mostly a 1:1 translation, except for {x} / {x,} / {x,y} quantifiers for which we define sub-rules to keep the output lean. ^$z,Pattern must start with "^" and end with "$"r9r=rsrc$|\}}|rd|zdzS|S)Nr7r)rtxt is_literals rto_rulez/SchemaConverter._visit_pattern..to_rule s! ! S*&039s? 9c 9rcr}gfd}fd}krv}|dk(rj|dfdz nF|dk(rJdz krdk7sJdd d d jdd dfn|d k(r,dz |d kDr |dz dk(sJd|dd|S|dk(r||}dz kr<dk7r4dk(r|dzz }dz n |z }dz kr dk7r4ksJd|dd|dz }dz j|dfnE|dk(rjddz n(|dvrd|zdfd<dz n |dk(r?|}dz kr#dk7r|z }dz kr dk7rksJd|dd|dz }dz |ddjdDcgc]}|j}}d }d} t|dk(rt |d }|} n:t|dk(sJ|d rt |d nd }|drt |dnd} d\} } | s=j | } | (jd!tdz| } | | <| } t| rd"| d"n| || | #dfd<nd$} krdk(r<dz kr4dz}|tvrdz | z } dz np| dzz } dz n_d"k(rjs | d%z } dz n@tvr4dz k(s| d$k(sdzdk(sdztvr| z } dz nnkr| rj| d&fkrv|Scc}w#t $rt d|d d wxYw)'z Parse a unit at index i (advancing it), and return its string representation + whether it's a literal. cZjrt}nt}jd|S)Nrr)ryDOTALLDOTr)rrs rget_dotzBSchemaConverter._visit_pattern..transform..get_dots'<<!DD~~d33rc g}tdD]B\}}|r*|jdjd|Ddf2|j|Dt |dk(r|dSdjfdDd fS) Nc |dS)Nr9r)rs rrzUSchemaConverter._visit_pattern..transform..join_seq..(s AaDrr<c3&K|] }|d yw)rNr)rdrs rrfzVSchemaConverter._visit_pattern..transform..join_seq..*s+c3.K|] }|ywrr)rdrrs rrfzVSchemaConverter._visit_pattern..transform..join_seq../s 9 9sF)rappendrLextendr)retrgseqrs rjoin_seqzCSchemaConverter._visit_pattern..transform..join_seq%s%,S.%A&MJ! BGG+.transformsNE +-C 4 Bf*AJ8JJ 512FA#XFA6z#AJ#-d9'!*[QRPSSXY`Xaabcd-JJ!GIK$8#9 ;UCD#XFA geai&8C&?`9%qcV]U^_`?#:%#X&'OFAf*s):"1:-+wq1q5/AAOFA+wqz9OFA f*s):F d=eWF1#\ZaYbcd"#s*OFAJJ78#XJJ|,FA/)&s2w/!3U;CGFA#X%&NFAf*s):&'!*4Qf*s):F c(+DG I(1I#&t9>1>8D#'MM !Q '71: 5 !Q '71q1u+= = !Q$QZ3.t7H7H#u,GFA$QZ>!O&"}&q1u~4&q1u~_D#wqz1GFA!-f*. GT?3_f*b: }P&(1.1AwiqQs'NANN6rbrk)rendswithrr r1r3rrz)rrrrrrrrs```@@@@@r_visit_patternzSchemaConverter._visit_patterns!!#&7+;+; ,  : 9 : !B- W :uS$Y' :C :T 5d+T T l~~ $$ $   44~E   rc|jdd}||jvre||jvrW|jj||j|}|j ||}|jj ||S)Nrr=)rr|raddr}rremove)rrref_nameresolveds r _resolve_refzSchemaConverter._resolve_refsw99S>"% 4;; &3d6O6O+O  % % ) )# .zz#Hzz(H5H  % % , ,S 1rcJ|jtj|Sr)rrdumps)rras r_generate_constant_rulez'SchemaConverter._generate_constant_rules##DJJu$566rc  |jd}|jd}tvrdznxsd}|jdx}!j|j|Sd|vsd|vr8j|j |jdxs|dSt |t r2j|j |Dcgc]}d|ic}Sd|vr$j|j|dSd |vr/d jfd |d D}j||S|d vrd |vs d|vr~|ddurwt|jdgt |jd ijj|j|jdS|d vrkd|vrgtg} fd} |dD]%}d|vr|dD] } | | d| |d'j|j| gS|dvrd|vsd|vr|jdxs|d} t | t r:j|ddjfdt| DzdzSj| rdndd} |jdd }|jd!}j|dt| ||d"#zdzS|d$vrd%|vrj|d%|S|d$vr:t!j"d&|xsdr j%|dk(rdn|t&d'S|d$vr9|d(t(vr.|d(}j|j%|t(|S|d)k(rgd*|vsd+|vr_j%d,t&d,}|jd*d }|jd+}j|d-t|||zd.zS|d/k(st+|d k(r)j|j%d/t&d/S|t&vs Jd0|j%|dk(rdn|t&|Scc}w)1Ntypeformatrr roneOfanyOfconstenumrc3@K|]}j|ywr)r)rdrrs rrfz(SchemaConverter.visit..sW1t;;A>Ws)Nr[ propertiesadditionalPropertiesTrequiredallOfc|jdx}j|}d|vrC|djD],\}}j||f|sj |.yy)Nrr)rr}itemsrr) comp_schema is_requiredr prop_name prop_schemarrrs r add_componentz,SchemaConverter.visit..add_componentsu&??622C?"&**S/K;.2=l2K2Q2Q2S4. ;"))9k*BC&$LL34/rF)r )additional_properties)Nr\r  prefixItemsz "[" space "," space c3^K|]$\}}j|rdndd|&yw)rr<ztuple-Nr)rdritemrrs rrfz(SchemaConverter.visit.. s<)#At 4D6#21FfQC)PQ)s*-z "]" spacer<rminItemsrmaxItemsz "," space)rI)Nr]rz ^uuid[1-5]?$rnz-stringr] minLength maxLengthrlrbrkr[zUnrecognized schema: )rRESERVED_NAMESrrrrrWrrLr~r _build_object_rulerrrRrrematch_add_primitivePRIMITIVE_RULESSTRING_FORMAT_RULESr)rrr schema_type schema_format rule_namertr hybrid_namerttr item_rule_namerMrN prim_name char_rulemin_lenmax_lenrrs` ` @@rrzSchemaConverter.visits=jj(  8, "&."8D3Jdnf ::f% %C 2>>)T->->s-CD D  'V"3>>))$ 70C0VvgW   T *>>))$k0R&!0RS   >>477wH v ::WvWXD>>)T2 2 , , F "&&012$>6::j"56Hfjjr:@@BCJ>>''$ ;Q0R , ,F1BuHJK 4G_ 7a<j=%be<="!6  7>>''+R( O + v &!8JJw'@6-+@E%&~~ #(()'0'7) # #"&EdV4CR;PPT3U!V"JJz15 "JJz2 ~~ '& 9[# #, ,f1D&&vi'8)D D , , ]0b2 &&#v-='  + + /)-@@(/1I>>##I/B9/MN  H $ 6 ![F%:++FOF4KLIjja0Gjj-G>>#Iw@A ! X%3v;!+;>>4..x9RS  /1 S5J6(3S S1&&#v-; , 1Ss Rrrc|j||j}|jD]\}tj |xst j |}|s Jd|d||j vsK|j||^|S)NzRule z not known)rrFrUrrr r|r)rrrredepdep_rules rrzSchemaConverter._add_primitiveNs NN4 .99 3C&**3/O3F3J3J33OH 4uSE4 48$++%##C2  3 rrrrc jtt|fdDcgc] \}}|d }}}i|D]h\}} j| rdnd|} j rdnd|dj t j|d| |<j|D cgc] } | |vs|  } } |D cgc] } | |vs|  c} |dk(st|trmrdndd } j|dk(rin|| d }j | djd td d |zd <jd d}|djfd| Dz }rP|dz }| r|dz }fd|djfdttDz }| r|dz }|dz }|dz }|Scc}}wcc} wcc} w)NcPj|ddt|dfS)Nr9r)rr)ikvrs rrz4SchemaConverter._build_object_rule..es(Aq 3z?!KSQRV Trrrrr<z-kvz space ":" space T additionalz-valuer]z ":" space rKz "{" space rc3(K|] }| ywrr)rdkprop_kv_rule_namess rrfz5SchemaConverter._build_object_rule..s"QQ#5a#8"Qsrz "," space ( c |^}}|}|dk(r% jrdndd|d|zdz}n |rd|d}n|}t|d kDr+|d jrdnd|d |d zz }|S)NrKrr<zadditional-kvsz ( "," space z )*z ( "," space  )?rr>z-restTfirst_is_optional)rr) ksr:r5rest kv_rule_nameresget_recursive_refsrr6rs rr?z>SchemaConverter._build_object_rule..get_recursive_refss T1!4 8..&" 5^D'. 6EMC'(c:C&Ct9q=3&" 5aS>*44H"C rrc38K|]}|ddyw)NFr9r)rdrr?optional_propss rrfz5SchemaConverter._build_object_rule..s*#>!"#5OOsz )r8z "}" space)rwsortedrrrrrrrrrrrrLranger)rrrrrrkv sorted_propsrrprop_rule_namer5required_propssub_name value_rulerr?rAr6rs` ` @@@@rrz"SchemaConverter._build_object_ruleXsN%%  *%T 2 qE   &0  "I{!ZZvTcr%:9+FN-1NN&"-i[<((I)>?@@QR`Qab- y )  &2CQ(]!CC%1GQh5F!G D (J7Ld,St4J?H+t39N*F#J'+nn*C ##Hoh.GH |,-' s #  ! !# & """Q."QQQ  DLD' & EJJs>23 D  EMD  M "DGsG4 G>G GGczdjdt|jjdDS)Nr6c30K|]\}}|d|yw)z ::= Nr)rdrrs rrfz1SchemaConverter.format_grammar..s' dfE$  sc |dSrr)rDs rrz0SchemaConverter.format_grammar..s Arr2)rLrBr|r )rs rformat_grammarzSchemaConverter.format_grammars5yy $T[[%6%6%8>NO   rN)TF)r'r/r0rrr1r3rrrrrrrrrrTrrr rrr rrMrrrrtrt|s *LQ00$(0 04 343c3j w r7Sj3kOsCx)Oc(O O %T3Y/ Ob rrtrrc|xsg}tj|}t|Dcic]\}}|| }}}t|ddd}|j |d}|j |d|j Scc}}w)NF)rrrrstdinr<)rloadsrrtrrrM)rridxr converters rr-r-s!rJ ZZ F-6z-BC T$)CJC5EI # #FG 4F OOFB  # # %% Ds A=)NFr),__doc__pathlibr itertoolsrtypingrrrrr r rr ARITHMETIC_GBNFC_GBNF CHESS_GBNF JAPANESE_GBNF JSON_ARR_GBNF JSON_GBNF LIST_GBNFrrr{compilerrrrRrT_up_to_15_digitsrLrr rrr~keysrrrrtr1r-rrrrasL$RR>:+ Z 8 8# L  8  j ! # #45&BJJz2!&e%@ PU5p %Wa45r:+; ;R@ !14D!DbIT .)7/9JK ;B] 7;gY  ,,C2BC C D     \  2VH=  +? H k    a  _vv.>?7&B7&B#$AK=Q  % UI_))+I.A.F.F.HI m$),\):&q q h & &$s)1D &r