sg3 <ddlZddlmZddlmZddlmZdZedejdejd ejfd Zedejdejd ejfd Z Gd d ejjZ GddZ y)N)jit)language)next_power_of_2c4|dkry|dkry|dkry|dkryy ) Nii)ns Q/var/www/html/venv/lib/python3.12/site-packages/triton/ops/blocksparse/softmax.py num_warpsrs-CxCxCxDy ROW_SIZE BLOCK_SIZEIS_DENSEc tjd} tjd}tjd}| tjdz|z}tjd| | z}tjd| | z}||| zdzz}tj|dz}tj|dz}||z}|||z| z| zz }||| z| zz }| rtjd| }n]|dtjdztjdz| zz}tj||z|z||kd}|| z|z}||k}tj||z|z|t d }|j tj}|}||z}|L|||zz }|| |zz }||z dz |z}|dk\||kz}tj|||zz|z|d}||z }|j tj}tj||kD| zt d |}tj|}tj||z|z||y)Nrr r maskotherinfr) tl program_id num_programsarangeloadfloattofloat32wheresoftmaxstore) OutA stride_xzLUTRextent stride_zr stride_hrscale is_causalrrrhmzhmlane_nblock_nheadersizeoffsetoff_ansoff_lutstart_nraoutoff_lomask_lo rel_logitss r_blocksparse_softmax_fwdrEsZ aA aA aA R__Q  ! #B YYq( #j 0Fii8$ 2G B*$) )F 776A: D WWVaZ F ME fw* ,z 99E a*n **E YYq( #1rq11BOOA4FF*TT''#-'1$aP z !F * T>D E F"eEl]CA RZZA C5LC} Q] Q]1*q.B&Q;6F?3WWQV^f47#N  z && C ((BFi'%, r?r@doutdarBrCDAss' r_blocksparse_softmax_bwdrUJs aA aA aA R__Q  ! #B YYq( #j 0Fii8$ 2G B*$) )F 776A: D WWVaZ Fw* ,z 9F q:~++F T>D q;  'B 1|# #f ,E YYq( #1rq11BOOA4FF*TT''#-'1AF z !F * V $c2A RZZA 7756>C 8D 772:: D "q&I%a0"a8A dRVVAHa(( )B ~ a)m a)m1*q.B&Q;6F?3d: a&j6)2G< eB q:~  &CHHS6\2D)rc<eZdZedZedZedZy)_softmaxc tjgtj|j}|j }t |j dD]2}tj|||ddddfjdf}4||z}tj|}tj|ddd|dd|jddddf}tj||fdjd} tj| |fjtjj!|} | t#|j%fS) Ndtypedevicer)dimr F)as_tupler )torchtensorint64r[clonerangeshapecatrG zeros_likecumsumnonzerostackviewtypeint32r$intmax) layoutblockr[_emptysizesr3 total_sizesoffsetscolumnsr9luts rmake_lutz_softmax.make_lutsb FMMJ v||A' @AIIufQ1Wo&9&9"&=>?E @em ""5)ll5":15 ..%.0A6eW-15::2>ii)*// <??GC )***rc |Et|tjr+|jjdk(sJ|j }|j d} |d|d|z| g} |dn |j } |dn|j} tj|}t| |||jd||| d| d| d|||t|| t||j||||_ ||_||_||_| |_| |_|j&|_| |_||_|S)Ncpurr )r r r r r\rrrr) isinstancer_Tensorr[rkitemrdstride empty_likerErrsave_for_backwardspdimsrpmaxlutr1 rel_shape rel_stridesrZ rel_dtypeis_denser2)ctxr@r1rDr2rrprvrrMgridrrrAs rforwardz_softmax.forwards>  E5<O>O>Q q! & AHHQK  " {1~{1~  $V,'  c3'    ! %  !  rc|j\}}d}|jdr6tj|j|j |j }|jd}|jd|jd|jz|f}tj|}t|||jd||jd||jd|j|||jd|jd|jd|jd|j|jt!|j"|j$t'|j"|dd|ddddddddddddddfS)NrrYrr r\r rz) saved_tensorsneeds_input_gradr_zerosrrr[rdrrprrUr~r1rr2rrrr)rrRrArvdrrrrSs rbackwardz_softmax.backwardsT$$S     "S]]#-- SB IIaL 1 szz!}syy8!<   d # &  ! $++a. A II   b!3??1#5sq7I3??[\K] MMyy$SZZ0\\ + D$D$dD$dTXZ^`dfjlprvwwrN)__name__ __module__ __qualname__ staticmethodrwrrrrrrWrWs?++   DxxrrWc$eZdZddZdddddZy)r'Fc|j|_||_||_tj |j|j|\|_|_||_y)N) rdrrorprWrwrvrr)selfrorpr[rs r__init__zsoftmax.__init__sGll   ( 1 1$++tzz6 R$+  rg?N)r1rDr2c  |1|j|jk7rtd|jtj|||||j|j |j |j|j }|S)Nz$relative position embedding must be ) rZ ValueErrorrWapplyrrprvrr)rr@r1rDr2s r__call__zsoftmax.__call__sp  !j&6&6!''&ACAGG9MN N NN1eZDKKUYU]U]_c_j_j== *r)F)rrrrrrrrr'r's!$'45rr')r_rrrrr constexprrErUautogradFunctionrWr'rrrrs  23(*||23*, 23 (*|| 2323j6*(*||6**,6*(*||6*6*rOxu~~&&Oxdr