
    sgЇ                    ^:   U d dl mZ d dlZd dlZd dlmZ d dlmZmZ d dl	m
Z
 d dlmZ d dlZd dlZd dlmZ d dlmc mZ d dlmZ d dlmZmZmZmZmZmZmZmZ d d	lm Z m!Z! d d
l"m#Z#m$Z$ d dl%m&Z& d dl'm(Z( d dl)Zd dl*m+Z+m,Z,m-Z-m.Z.m/Z/m0Z0m1Z1 ejd                  Z2dZ3d Z4d Z5 e6ddddd ddejn                         e6dddddd ddejn                  	       e6ddd ejn                  !       e6dd"d#d$d%d ejn                  &       e6d'd(d) ejn                  *       e6d+d,d-d.d d ejn                  /      gZ8d0 Z9d1 Z:d2 Z;d3 Z<d4 Z=d5 Z>d6 Z?d7 Z@d8 ZAd9 ZBd: ZCd; ZDd< ZEd= ZFd> ZGd? ZHd@ ZIdA ZJdB ZKdC ZLdD ZMdE ZNdF ZOdG ZPdH ZQdI ZRdJ ZSdK ZTdL ZUdM ZVdN ZWdO ZXdP ZYdQ ZZdR Z[dS Z\dT Z]dU Z^dV Z_dW Z`dX ZadY ZbdZ Zcd[ Zdd\ Zed] Zfd^ Zgd_ Zhd` Zida Zjdb Zkdc Zldd Zmg  e;        e<        e>        e@        eA        e=        e?        eB        eC        eD        eE        eF        eG        eH        eI        eJ        eK        eL        eM        eN        eO        eP        eQ        eR        eS        eU        eT        eV        eX        eW        eY        eZ        e[        e\        e]        e^        e_        e`        ea        eb        ec        ed        ee        ef        eg        eh        ei        ej        ek        el        e6dedfdgdhdddejn                  i       e6dedjdkdhddlddejn                  m	       e6dedndodhddpddqejn                  m	       e6dedrdsdhddtddejn                  m	       e6dedudvdwddxddejn                  m	       e6dedydzdwdd{ddejn                  m	       e6dedfdgd|dd}dd~       e6dd ddhddejn                         e6dd dddddejn                         e6dd ddhdddejn                         e6dd ddhdddejn                         e6dd ddhdddejn                         e6dd ddhdddejn                         e6dd dddddejn                         e6ddddddddejn                  m	       e6ddddddddejn                  m	       e6dd dddddejn                         e6ddddddddejn                  	       e6dddd.dddddejn                  
       e6dddd.dddddejn                  
       e6dddddddddejn                  
       e6dddddddddejn                  
       e6dddddd}dd       e6dd ddddddejn                  	       e6dd dddddejn                         e6dd dddddejn                         e6ddń dddddqejn                         e6ddȄ dddddqejn                         e6ddddddddqejn                  Ϋ	       e6dddddddddqejn                  
       e6dddddddddqejn                  
       e6ddՄ ddddddqejn                  	       e6ddل ddddejn                         e6dd݄ ddddejn                         e6dd ddddejn                         e6dd ddddejn                         e6dd ddddejn                         e6ddddddddejn                  	       e6dddddddddejn                  
       e6ddddddd ddejn                  
       e6ddddddldddejn                  
       e6dddddddddejn                  
       e6ddddddd}d       e6dd ddddddejn                  	       e6d d ddddejn                         e6dd ddddejn                         e6dd dd	dddejn                         e6d
d dd	dddejn                         e6dd dd	dddejn                         e6dddddddejn                         e6ddddddddejn                  	       e6ddddejn                         e6ddddemdejn                         e6dddd dd        e6d!d"d#d$ d ejn                   ed%      &       e6d!d"d#d' d d(ejn                   ed%      )       e6d*d"d+d, d d-ejn                  .       e6d*d"d+d/ d d(ejn                  .       e6d*d0d1d2 d d3ejn                  .       e6d*d4d5d6 d d7ejn                  .       e6d8d9 d:d; d ejn                  <       e6d=d> d?d@ d ejn                  <       e6dAdB dCdD d ejn                  <       e6dEdF dGdH d dI       e6dJ dKdL dMd dN       e6dOdPdQdRejn                         e6dSdPdTdUejn                         e6 e:ej                  dVddWX      dYdZd[d ejn                  \       e6 e:ej                  dVddWX      dYd]d^d _       e6 e:ej                  d`ddWX      dYdadbd ejn                  \       e6 e:ej                  ddcdWX      dddZded ejn                  \       e6 e:ej                  dVddfd g      dhdZdid ejn                  \       e6 e:ej                  djddfd g      dkdadld ejn                  \       e6 e:ej                  ddcdfd g      dmdZdnd ejn                  \       e6 e:ej                  dVddfd g      dhd]dod _       e6 e:ej                  dVddfdg      dpdZdqd ejn                  \       e6 e:ej                  ddcdfdg      drdZdsd ejn                  \       e6 e:ej                  dtddWX      dudvdwd ejn                  \       e6 e:ej                  dVddWX      dxdydzd ejn                  \       e6 e:ej                  d{ddWX      d|d}d~d ejn                  \       e6 e:ej                  ddcdWX      ddydd ejn                  \       e6 e:ej                  dVddWX      dxddd _       e6 e:ej                  dVddd g      ddydd ejn                  \       e6 e:ej                  dVddd g      dddd _       e6 e:ej                  dddd g      dddd ejn                  \       e6 e:ej                  ddcdd g      ddydd ejn                  \       e6 e:ej                  dddd g      ddydd ejn                  \       e6 e:ej                  dddd g      ddydd ejn                  \       e6 e:ej                  ddddg      ddydd ejn                  \       e6 e:ej                  ddddg      ddydd ejn                  \       e6 e:ej                  dVddd g      ddydd ejn                  \       e6 e:ej                  dVddd g      dddd _       e6 e:ej                  dddd g      dddd ejn                  \       e6 e:ej                  ddcdd g      ddydd ejn                  \       e6 e:ej                  dddd g      ddydd ejn                  \       e6 e:ej                  dddd g      ddydd ejn                  \       e6 e:ej                  ddddg      ddydd ejn                  \       e6 e:ej                  ddddg      ddydd ejn                  \       e6 e:ej                  dVddWX      dddd ejn                  \       e6 e:ej                  dVddWX      dddd _       e6 e:ej                  dddWX      dddd ejn                  \       e6 e:ej                  ddcdWX      dddd ejn                  \       e6 e:ej                  dVddd g      dddd ejn                  \       e6 e:ej                  dVddd g      dddd _       e6 e:ej                  dddd g      dddd ejn                  \       e6 e:ej                  dddd g      ddddd ejn                         e6 e:ej                  ddddg      dddd ejn                  \       e6 e:ej                  ddddg      ddddd ejn                         e6 e:ej                  dÐī      dŐdƐdd ejn                  \       e6 e:ej                  dej                  ɫ      dʐdƐdd d ejn                  ̫       e6 e:ej                  dȐī      d͐dΐdd ejn                  \       e6 e:ej                  dȐī      d͐dАdd ejn                  \       e6 e:ej                  dej                  ɫ      dʐdАdd d ejn                  ̫       e6 e:ej                  d ī      dd(dd d ejn                  ի       e6 e:ej                  d֐ī      dd(dd d ejn                  ի       e6 e:ej                  dÐī      dŐdِdd d ۫       e6 e:ej                  dÐī      dܐdƐdd ejn                  \       e6 e:ej                  dȐī      dސdΐdd ejn                  \       e6 e:ej                  dȐī      dސdАdd ejn                  \       e6 e:ej                  d ī      dd(dd ejn                  \       e6 e:ej                  d֐ī      dd(dd ejn                  \       e6 e:ej                  d ī      ddِdd _       e6dd ddd dejn                         e6dd ddd dejn                         e6dd ddd emdejn                         e6dd ddd dejn                         e6dd ddd dejn                         e6dd ddemd dejn                         e6dd"d#dِdd ejn                  &       e6dd ejn                         e6dd  dejn                         e6dd dejn                         e6dddd dejn                  	       e6dd
 emdejn                         e6dddddddd d ejn                  
       e6dddtddej                  fddd dde!rdndejn                  	       e6dddd d dddejn                  	       e6dddtddej                  fdd  d dddejn                  	       e6d!ddtdtdtddej                  fd"d# d d$de!rdnd%ejn                  	       e6dd&d'd( d) dddejn                  *	       e6d'd+d,d-emdejn                  .       e6d/d0d1 ej                  dtdtg      fd-emdejn                  .       e6d2g d3dd fd4d5ddddd67	      Zu evg d8g d9      D ]  \  ZwZxdaD ]  Zyeydk(  rewd:k(  r ez e{deydz               Z|d;d<j                   e~ee|            z   d=z   Zd>djeyz  z   Zd? ezd@ e|D              z   Zeuj                   e6dAey dBdtd֐d֐dte|dȐddewf	dCey dDe dEex dFeedew dGddejn                  H
               g dIZi dJdKejn                  dLdMdNdidOd d ejn                  dPdQd d ejn                  dPdd ejn                  dRdSdTejn                  idUdTejn                  idVdTejn                  idWdTejn                  idXdTejn                  idYdTejn                  idZdTejn                  id[dTejn                  id\dTejn                  id]dTejn                  id^dTejn                  id_dTejn                  idTejn                  idTejn                  idTejn                  idTejn                  idTejn                  idTejn                  idTejn                  id`Ze+ee6f   eda<   eD ]I  Z e6edjemdd b      Zej                  ei       Zej                  e       euj                  e       K ddcZ	 	 dddZ	 	 ddeZ	 	 ddfZ	 	 ddgZ	 	 ddhZddiZddjZdk ZddlZddmZddnZdo ZddpZddqZ	 	 ddrZddsZddtZe eedu      eeeeeeeeeeeeedvZe+dwe,f   edx<   g Zdy Zg dzZg d{Z e
ee      D ]?  \  ZZ e6e d|e ed}d~dPdPed ejn                        Zej                  e       A eD ]8  Z e6de d d d ed ejn                        Zej                  e       : dd d fdd d fdd d fdd d fdd d fdd d fdd d fdd d fdd d fdd d fg
Zddd iiZe+ee6f   ed<   d d d d d dZg d{Z e
ee      D ]u  \  \  ZZZZ e6e d|e ed}defdefdedej                  ed            Zej                  ei       Zej                  e       ej                  e       w  G d de      Z G d d      Z G d de      Z G d d      Z G d dee      Z G d dee      ZdÐdZdĐdZd Zy(      )abstractmethodN)deepcopy)reducepartial)product)mul)
_reduction)TestCaseto_gpufreeze_rng_stateis_iterable	gradcheckgradgradcheckset_default_dtypeskipIfTorchDynamo)	TEST_CUDASM90OrLater)_get_numerical_jacobian_iter_tensors)Variable)_TensorOrTensors)DictCallableTupleListSequenceUnionAnyh㈵>c                 r    t        | dd       }|#t        j                  t        | dd       dd      }|J |S )N	reductionsizeAverageTF)emit_warning)getattr
_Reductionlegacy_get_stringmresults     T/var/www/html/venv/lib/python3.12/site-packages/torch/testing/_internal/common_nn.pyget_reductionr+       sD    QT*F~--ga.MtbghM    c                 >    t        | dd       }||S t        | dd       S )Nweightweights)r$   r'   s     r*   
get_weightr0   (   s*    Q$'F1i&&r,   Linear)
      ztorch::nn::LinearOptions(10, 8))   r2   c                     t        j                  | |d   j                               |d   j                  dd      j	                  dd      z   S )Nr      r4   r3   )torchmmtviewexpandip_s      r*   <lambda>rA   k   s>    UXXa1%:QqTYYq"=M=T=TUVXY=Z%Z r,   T{Gzt?)module_nameconstructor_argscpp_constructor_args
input_sizereference_fn	with_tf32tf32_precisiondefault_dtype)r2   r3   Fz+torch::nn::LinearOptions(10, 8).bias(false)no_biasc                 P    t        j                  | |d   j                               S Nr   )r8   r9   r:   r=   s      r*   rA   rA   v   s    UXXa1%: r,   )	rC   rD   rE   rF   descrG   rH   rI   rJ   RReLU)r6      rP   F)rC   rF   	test_cudarJ   )皙?g?z/torch::nn::RReLUOptions().lower(0.1).upper(0.9))r4   r4      with_up_down)rC   rD   rE   rF   rN   rQ   rJ   FlattenrP      r4   rS   c                 .    t        j                  | d      S Nr6   )r8   flatten)r>   r@   s     r*   rA   rA      s    5==A#6 r,   )rC   rF   rG   rJ   CrossMapLRN2d)rS   rB   MbP?rP   z>torch::nn::CrossMapLRN2dOptions(5).alpha(5e-3).beta(1e-3).k(2))rP   rW      r]   )rC   rD   rE   rF   check_gradgradcheck_batched_gradrJ   c                      t        t        | d      } t        j                  |      j                  |  j                         S rY   )r   r   r8   randpermr;   double)sizetotals     r*   _rand_tensor_non_equalre      s5    3a E%5>>% %%t,3355r,   c                 B      G  fddt         j                        }|S )Nc                       e Zd Z fdZy))wrap_functional.<locals>.FunctionalModulec                      |i S N )selfargsfnkwargss     r*   forwardz1wrap_functional.<locals>.FunctionalModule.forward   s    t&v&&r,   N)__name__
__module____qualname__rp   )rn   ro   s   r*   FunctionalModulerh      s    	'r,   rt   )nnModule)rn   ro   rt   s   `` r*   wrap_functionalrw      s    '299 ' r,   c            
           t        j                  dd       t        dt         fd      dd d d fdd	t         j                  
      S )Nr2   PoissonNLLLoss_no_reducec                 R    t        j                  | j                  |       d      S Nnoner!   )Fpoisson_nll_losstype_asr>   r:   s    r*   rA   z/poissonnllloss_no_reduce_test.<locals>.<lambda>       a((AIIaLFK r,   zaF::poisson_nll_loss(i, t.to(i.options()), F::PoissonNLLLossFuncOptions().reduction(torch::kNone))c                  .    t        j                  dd      S Nr2   r8   randrk   r,   r*   rA   z/poissonnllloss_no_reduce_test.<locals>.<lambda>   s    B+ r,   _get_input()r   c                 H    | j                         j                  |       z
  S rj   )expr   r>   r@   r:   s     r*   rA   z/poissonnllloss_no_reduce_test.<locals>.<lambda>   s    1557QUU1X#5 r,   Ffullnameconstructorcpp_function_callinput_fncpp_var_maprG   picklerJ   r8   randndictrw   rb   r:   s   @r*   poissonnllloss_no_reduce_testr      sM    BA+#KMj+(q15ll
$ 
$r,   c                      t        t        j                  dd      j                  d      j	                  t        j
                               t        dt         fd      dd d d	 fd
ddt        j
                  	      S )N   r2   r   BCELoss_no_reducec                 R    t        j                  | j                  |       d      S r{   r~   binary_cross_entropyr   r   s    r*   rA   z(bceloss_no_reduce_test.<locals>.<lambda>       a,,Q		!O r,   iF::binary_cross_entropy(i, t.to(i.options()), F::BinaryCrossEntropyFuncOptions().reduction(torch::kNone))c                  N    t        j                  dd      j                  dd      S Nr   r2   y&1?v?r8   r   clamp_rk   r,   r*   rA   z(bceloss_no_reduce_test.<locals>.<lambda>       B+226:F r,   r   r   c                 `    | j                         z  dz
  d| z
  j                         z  z    S rY   logr   s     r*   rA   z(bceloss_no_reduce_test.<locals>.<lambda>   ,    Q[AEa!e[[]3J%J#K r,   FgǺF?	r   r   r   r   r   rG   r   	precisionrJ   )r   r8   r   gttorb   r   rw   r   s   @r*   bceloss_no_reduce_testr      so    R$''*--ell;<A$#OQnF(q1Kll$ $r,   c            
           t        j                  d      j                  d      j                  t         j                         t        dt         fd      dd d d fd	d
t         j                        S )Nrk   r   BCELoss_no_reduce_scalarc                 R    t        j                  | j                  |       d      S r{   r   r   s    r*   rA   z/bceloss_no_reduce_scalar_test.<locals>.<lambda>   r   r,   r   c                  L    t        j                  d      j                  dd      S Nrk   r   r   r   rk   r,   r*   rA   z/bceloss_no_reduce_scalar_test.<locals>.<lambda>       B..vzB r,   r   r   c                 `    | j                         z  dz
  d| z
  j                         z  z    S rY   r   r   s     r*   rA   z/bceloss_no_reduce_scalar_test.<locals>.<lambda>   r   r,   Fr   )r8   r   r   r   rb   r   rw   r   s   @r*   bceloss_no_reduce_scalar_testr      sd    B1  .A+#OQnB(q1Kll
$ 
$r,   c                  v    t        t        j                  ddt        j                        j	                  d      j                  t        j                               t        j                  dt        j                        t        dt         fd      dd d	 d
 fdddt        j                  	      S )Nr   r2   dtyper   BCELoss_weights_no_reducec                 r    t        j                  | j                  |       j                  |       d      S Nr|   r.   r!   r   r>   r:   r/   s    r*   rA   z0bceloss_weights_no_reduce_test.<locals>.<lambda>   .    a,,Q		!4;OOA4FRXZ r,   zF::binary_cross_entropy(i, t.to(i.options()), F::BinaryCrossEntropyFuncOptions().weight(weights.to(i.options())).reduction(torch::kNone))c                  N    t        j                  dd      j                  dd      S r   r   rk   r,   r*   rA   z0bceloss_weights_no_reduce_test.<locals>.<lambda>   r   r,   r   r   c                 f    | j                         z  dz
  d| z
  j                         z  z    z  S rY   r   )r>   r?   r(   r:   r/   s      r*   rA   z0bceloss_weights_no_reduce_test.<locals>.<lambda>   s2    q1557{a!eA{{}5L'L%MPW%W r,   Fa2U0*3?r   )	r   r8   r   rb   r   r   r   r   rw   r:   r/   s   @@r*   bceloss_weights_no_reduce_testr      s    R5<<8;;A>AA%,,OPAjj5<<0G,#Z[x G(qWEWll r,   c            
      @    t        j                  d      j                  d      j                  t         j                         t        j
                  dt         j                        t        dt         fd      dd dd	  fd
dt         j                        S )Nrk   r   r    BCELoss_weights_no_reduce_scalarc                 r    t        j                  | j                  |       j                  |       d      S r   r   r   s    r*   rA   z7bceloss_weights_no_reduce_scalar_test.<locals>.<lambda>   r   r,   zF::binary_cross_entropy(
            i, t.to(i.options()),
            F::BinaryCrossEntropyFuncOptions().weight(weights.to(i.options())).reduction(torch::kNone))r   r   c                  L    t        j                  d      j                  dd      S r   r   rk   r,   r*   rA   z7bceloss_weights_no_reduce_scalar_test.<locals>.<lambda>   r   r,   c                 f    | j                         z  dz
  d| z
  j                         z  z    z  S rY   r   r>   r@   r:   r/   s     r*   rA   z7bceloss_weights_no_reduce_scalar_test.<locals>.<lambda>   s1    Q[AEa!e[[]3J%J#Kg#U r,   F)r   r   r   r   r   rG   r   rJ   )r8   r   r   r   rb   r   r   rw   r   s   @@r*   %bceloss_weights_no_reduce_scalar_testr      s    B1  .Ajj5<<0G3#Z[k )qWEBUll r,   c                  0    t        t        j                  dd      j                  d      j	                  t        j
                              t        j                          t        dt        fd      dd dd	 fd
ddt        j
                  	      S )Nr   r2   r   BCEWithLogitsLoss_legacy_enumc                 R    t        j                  | j                  |       d      S )NF)r   r~    binary_cross_entropy_with_logitsr   r   s    r*   rA   z4bce_with_logistic_legacy_enum_test.<locals>.<lambda>  s    a88AIIaLQVW r,   F::binary_cross_entropy_with_logits(
            i, t.to(i.options()), F::BinaryCrossEntropyWithLogitsFuncOptions().reduction(torch::kNone))c                  N    t        j                  dd      j                  dd      S r   r   rk   r,   r*   rA   z4bce_with_logistic_legacy_enum_test.<locals>.<lambda>  r   r,   r   r   c                 x     |       j                         z  dz
  d |       z
  j                         z  z    S rY   r   r>   r@   sigmoidr:   s     r*   rA   z4bce_with_logistic_legacy_enum_test.<locals>.<lambda>  :    Q)9%9QUq7ST:~FZFZF\<\%\#] r,   F	r   r   r   r   r   rG   r^   r   rJ   
r   r8   r   r   r   rb   ru   Sigmoidr   rw   r   r:   s   @@r*   "bce_with_logistic_legacy_enum_testr     sz    R$''*--ell;<AjjlG0#WYkF(q1]ll r,   c                  0    t        t        j                  dd      j                  d      j	                  t        j
                              t        j                          t        dt        fd      dd dd	 fd
ddt        j
                  	      S )Nr   r2   r   BCEWithLogitsLoss_no_reducec                 R    t        j                  | j                  |       d      S r{   r   r   s    r*   rA   z2bce_with_logistic_no_reduce_test.<locals>.<lambda>      a88AIIaLTZ[ r,   r   c                  N    t        j                  dd      j                  dd      S r   r   rk   r,   r*   rA   z2bce_with_logistic_no_reduce_test.<locals>.<lambda>  r   r,   r   r   c                 x     |       j                         z  dz
  d |       z
  j                         z  z    S rY   r   r   s     r*   rA   z2bce_with_logistic_no_reduce_test.<locals>.<lambda>  r   r,   Fr   r   r   s   @@r*    bce_with_logistic_no_reduce_testr     sz    R$''*--ell;<AjjlG.#[]kF(q1]ll r,   c                      t        j                  d      j                  d      j                  t         j                        t        j                          t        dt        fd      dd dd fd	d
d
t         j                  	      S )Nrk   r   "BCEWithLogitsLoss_no_reduce_scalarc                 R    t        j                  | j                  |       d      S r{   r   r   s    r*   rA   z9bce_with_logistic_no_reduce_scalar_test.<locals>.<lambda>,  r   r,   r   c                  L    t        j                  d      j                  dd      S r   r   rk   r,   r*   rA   z9bce_with_logistic_no_reduce_scalar_test.<locals>.<lambda>/  r   r,   r   r   c                 x     |       j                         z  dz
  d |       z
  j                         z  z    S rY   r   r   s     r*   rA   z9bce_with_logistic_no_reduce_scalar_test.<locals>.<lambda>1  r   r,   Fr   )	r8   r   r   r   rb   ru   r   r   rw   r   s   @@r*   'bce_with_logistic_no_reduce_scalar_testr   &  sr    B1  .AjjlG5#[]kB(q1]ll r,   c                       t        j                  ddt         j                         t        dt	         fd      dd d d fd	d
dt         j                  	      S )Nr2   r   KLDivLoss_with_target_no_reducec                 R    t        j                  | j                  |       d      S r{   r~   kl_divr   r   s    r*   rA   z6kldivloss_with_target_no_reduce_test.<locals>.<lambda>=      ahhq!))A,&A r,   NF::kl_div(i, t.to(i.options()), F::KLDivFuncOptions().reduction(torch::kNone))c                  J    t        j                  dd      j                         S r   r8   r   r   rk   r,   r*   rA   z6kldivloss_with_target_no_reduce_test.<locals>.<lambda>?      B+//1 r,   r   r   c                 D    t        d   | j                  |       d      S N	KLDivLossr|   r}   loss_reference_fnsr   r   s     r*   rA   z6kldivloss_with_target_no_reduce_test.<locals>.<lambda>A  !    {+Aqyy|vN r,   TF	r   r   r   r   r   rG   supports_forward_adr   rJ   r8   r   rb   r   rw   r   s   @r*   $kldivloss_with_target_no_reduce_testr   8  sV    

2r.A2#ACj1(q1O ll$ $r,   c                       t        j                  ddt         j                         t        dt	         fd      dd d d fd	d
dt         j                  	      S )Nr2   r   KLDivLoss_no_reducec                 R    t        j                  | j                  |       d      S r{   r   r   s    r*   rA   z*kldivloss_no_reduce_test.<locals>.<lambda>M  r   r,   r   c                  J    t        j                  dd      j                         S r   r   rk   r,   r*   rA   z*kldivloss_no_reduce_test.<locals>.<lambda>O  r   r,   r   r   c                 D    t        d   | j                  |       d      S r   r   r   s     r*   rA   z*kldivloss_no_reduce_test.<locals>.<lambda>Q  r   r,   TFr   r   r   s   @r*   kldivloss_no_reduce_testr  H  sV    

2r.A&#ACj1(q1O ll r,   c                       t        j                  dt         j                         t        dt	         fd      dd d d fd	d
dt         j                  	      S )Nrk   r   KLDivLoss_no_reduce_scalarc                 R    t        j                  | j                  |       d      S r{   r   r   s    r*   rA   z1kldivloss_no_reduce_scalar_test.<locals>.<lambda>^  r   r,   r   c                  H    t        j                  d      j                         S Nrk   r   rk   r,   r*   rA   z1kldivloss_no_reduce_scalar_test.<locals>.<lambda>`      B++- r,   r   r   c                 D    t        d   | j                  |       d      S r   r   r   s     r*   rA   z1kldivloss_no_reduce_scalar_test.<locals>.<lambda>b  r   r,   TFr   r   r   s   @r*   kldivloss_no_reduce_scalar_testr
  Y  sT    

2U\\*A-#ACj-(q1O ll$ $r,   c                       t        j                  ddt         j                        j                          t	        dt         fd      dd d d fd	d
dt         j                  	      S )Nr2   r   #KLDivLoss_with_log_target_no_reducec                 T    t        j                  | j                  |       dd      S Nr|   T)r!   
log_targetr   r   s    r*   rA   z:kldivloss_with_log_target_no_reduce_test.<locals>.<lambda>n      ahhq!))A,&TR r,   _F::kl_div(i, t.to(i.options()), F::KLDivFuncOptions().reduction(torch::kNone).log_target(true))c                  J    t        j                  dd      j                         S r   r   rk   r,   r*   rA   z:kldivloss_with_log_target_no_reduce_test.<locals>.<lambda>p  r   r,   r   r   c                 D    t        d   | j                  |       d      S NKLDivLoss_log_targetr|   r}   r   r   s     r*   rA   z:kldivloss_with_log_target_no_reduce_test.<locals>.<lambda>r  #    56q!))A,RXY r,   TFr   r8   r   rb   r   r   rw   r   s   @r*   (kldivloss_with_log_target_no_reduce_testr  i  s_    

2r.224A6#RT{1(q1Z ll$ $r,   c                       t        j                  ddt         j                        j                          t	        dt         fd      dd d d fd	d
dt         j                  	      S )Nr2   r   KLDivLoss_no_reduce_log_targetc                 T    t        j                  | j                  |       dd      S r  r   r   s    r*   rA   z5kldivloss_no_reduce_log_target_test.<locals>.<lambda>~  r  r,   r  c                  J    t        j                  dd      j                         S r   r   rk   r,   r*   rA   z5kldivloss_no_reduce_log_target_test.<locals>.<lambda>  r   r,   r   r   c                 D    t        d   | j                  |       d      S r  r   r   s     r*   rA   z5kldivloss_no_reduce_log_target_test.<locals>.<lambda>  r  r,   TFr   r  r   s   @r*   #kldivloss_no_reduce_log_target_testr  y  s_    

2r.224A1#RT{1(q1Z ll r,   c                       t        j                  dt         j                        j                          t	        dt         fd      dd d d fd	d
dt         j                  	      S )Nrk   r   %KLDivLoss_no_reduce_scalar_log_targetc                 T    t        j                  | j                  |       dd      S r  r   r   s    r*   rA   z<kldivloss_no_reduce_scalar_log_target_test.<locals>.<lambda>  r  r,   r  c                  H    t        j                  d      j                         S r  r   rk   r,   r*   rA   z<kldivloss_no_reduce_scalar_log_target_test.<locals>.<lambda>  r  r,   r   r   c                 D    t        d   | j                  |       d      S r  r   r   s     r*   rA   z<kldivloss_no_reduce_scalar_log_target_test.<locals>.<lambda>  r  r,   TFr   r  r   s   @r*   *kldivloss_no_reduce_scalar_log_target_testr$    s]    

2U\\*..0A8#RT{-(q1Z ll$ $r,   c                       t        j                  dddt         j                         t        dt	         fd      dd d	 d
 fdddt         j                  	      S )NrP   rW   r4   r   L1Loss_no_reducec                 R    t        j                  | j                  |       d      S r{   r~   l1_lossr   r   s    r*   rA   z'l1loss_no_reduce_test.<locals>.<lambda>      aii199Q<6B r,   PF::l1_loss(i, t.to(i.options()), F::L1LossFuncOptions().reduction(torch::kNone))c                  0    t        j                  ddd      S NrP   rW   r4   r8   r   rk   r,   r*   rA   z'l1loss_no_reduce_test.<locals>.<lambda>      Q1- r,   r   r   c                 H    | j                  |       z
  j                         S rj   r   absr   s     r*   rA   z'l1loss_no_reduce_test.<locals>.<lambda>      A		!$4#9#9#; r,   TFr   r8   r   rb   r   rw   r   s   @r*   l1loss_no_reduce_testr5    sU    Aq!5<<0A##BDl-(q1; ll
$ 
$r,   c            
           t        j                  dddt         j                         t        dt	         fd      dd d	 d
 fddd      S )NrP   rW   r4   r   L1Loss_no_reduce_complexc                 R    t        j                  | j                  |       d      S r{   r(  r   s    r*   rA   z/l1loss_no_reduce_complex_test.<locals>.<lambda>  r*  r,   r+  c                  P    t        j                  dddt         j                        S )NrP   rW   r4   r   )r8   r   cdoublerk   r,   r*   rA   z/l1loss_no_reduce_complex_test.<locals>.<lambda>  s    Q1EMMB r,   r   r   c                 H    | j                  |       z
  j                         S rj   r1  r   s     r*   rA   z/l1loss_no_reduce_complex_test.<locals>.<lambda>  r3  r,   TF)r   r   r   r   r   rG   r   r   )r8   r   r:  r   rw   r   s   @r*   l1loss_no_reduce_complex_testr<    sN    Aq!5==1A+#BDlB(q1; 	 	r,   c                       t        j                  dt         j                         t        dt	         fd      dd d d fd	d
dt         j                  	      S )Nrk   r   L1Loss_no_reduce_scalarc                 R    t        j                  | j                  |       d      S r{   r(  r   s    r*   rA   z.l1loss_no_reduce_scalar_test.<locals>.<lambda>  r*  r,   r+  c                  ,    t        j                  d      S r  r.  rk   r,   r*   rA   z.l1loss_no_reduce_scalar_test.<locals>.<lambda>      R r,   r   r   c                 H    | j                  |       z
  j                         S rj   r1  r   s     r*   rA   z.l1loss_no_reduce_scalar_test.<locals>.<lambda>  r3  r,   TFr   r4  r   s   @r*   l1loss_no_reduce_scalar_testrC    sQ    Bell+A*#BDl((q1; ll
$ 
$r,   c                      d} t        j                  | dt         j                  it        dt	        fd      d| ddfdd	d
t         j                  	      S )NrV   r   MSELoss_no_reducec                 R    t        j                  | j                  |       d      S r{   r~   mse_lossr   r>   targets    r*   rA   z(mseloss_no_reduce_test.<locals>.<lambda>      ajjFNN1$5H r,   WF::mse_loss(i, target.to(i.options()), F::MSELossFuncOptions().reduction(torch::kNone))r   rI  c                 ,    | z
  j                  d      S NrP   powr>   r@   rJ  s     r*   rA   z(mseloss_no_reduce_test.<locals>.<lambda>      AJ#3#3A#6 r,   TF	r   r   r   rF   r   rG   r   r   rJ   r4  rF   rJ  s    @r*   mseloss_no_reduce_testrU    sY    J[[*9ELL9F$#HJs(F;6 ll
$ 
$r,   c                      d} t        j                  | t         j                        t        dt	        fd      d| ddfdd	d
t         j                  	      S )Nrk   r   MSELoss_no_reduce_scalarc                 R    t        j                  | j                  |       d      S r{   rG  rI  s    r*   rA   z/mseloss_no_reduce_scalar_test.<locals>.<lambda>  rK  r,   rL  r   rI  c                 ,    | z
  j                  d      S rN  rO  rQ  s     r*   rA   z/mseloss_no_reduce_scalar_test.<locals>.<lambda>  rR  r,   TFrS  r4  rT  s    @r*   mseloss_no_reduce_scalar_testrZ    sV    J[[5<<8F+#HJs(F;6 ll
$ 
$r,   c            
      (    t        t        j                  d      j                         j	                  d      j                         j                               ddi t        dt         fd      dd d	d
 fddt        j                        S )Nr   r2   r!   r|   NLLLoss_no_reducec                 t    t        j                  | j                  |       j                         d         S Nr!   r}   r~   nll_lossr   longr>   ro   r:   s    r*   rA   z(nllloss_no_reduce_test.<locals>.<lambda>  *    ajjAIIaL$5$5$76+CVW r,   pF::nll_loss(
            i, t.to(i.options()).to(torch::kLong), F::NLLLossFuncOptions().reduction(torch::kNone))c                  J    t        j                  dd      j                         S Nr   r2   r   rk   r,   r*   rA   z(nllloss_no_reduce_test.<locals>.<lambda>  r   r,   r   r   c                 ^    t        d   | j                  |       j                         fi S NNLLLossr   r   ra  r>   r@   ro   r:   s     r*   rA   z(nllloss_no_reduce_test.<locals>.<lambda>  -    y)!QYYq\->->-@KFK r,   Fr   
r   r8   emptyuniform_r   floorra  r   rw   rb   ro   r:   s   @@r*   nllloss_no_reduce_testrr    s    R))+//399;@@BCA6"F$#WYg1(q1Lll$ $r,   c            
      *    t        t        j                  d      j                         j	                  d      j                         j                               ddd t        dt         fd      dd	 d
d fddt        j                        S )Nr   r2   rP   r|   ignore_indexr!   NLLLoss_no_reduce_ignore_indexc           	          t        j                  | j                  |       j                         t	        d         t        d               S Nru  r!   rt  r~   r`  r   ra  intstrrb  s    r*   rA   z5nllloss_no_reduce_ignore_index_test.<locals>.<lambda>  @    ajjAIIaL$5$5$7c&Q_J`Fa+.vk/B+CE r,   zF::nll_loss(
            i, t.to(i.options()).to(torch::kLong), F::NLLLossFuncOptions().ignore_index(2).reduction(torch::kNone))c                  J    t        j                  dd      j                         S rf  r   rk   r,   r*   rA   z5nllloss_no_reduce_ignore_index_test.<locals>.<lambda>  r   r,   r   r   c                 ^    t        d   | j                  |       j                         fi S rh  rj  rk  s     r*   rA   z5nllloss_no_reduce_ignore_index_test.<locals>.<lambda>  rl  r,   Fr   rm  rq  s   @@r*   #nllloss_no_reduce_ignore_index_testr    s    R))+//399;@@BCA:;&)QF1#EFw1(q1Lll$ $r,   c            
      X    t        t        j                  d      j                         j	                  d      j                         j                               t        j                  d      fd t        dt         fd      dd dd	 fd
dt        j                        S )Nr   r2   c                 ,    j                  |       ddS r   r   r>   r.   s    r*   ro   z.nllloss_no_reduce_weights_test.<locals>.kwargs       ..+&AAr,   NLLLoss_no_reduce_weightsc                 x    t        j                  | j                  |       j                         fi  |       S rj   r_  rb  s    r*   rA   z0nllloss_no_reduce_weights_test.<locals>.<lambda>  ,    ajjAIIaL$5$5$7E6!9E r,   F::nll_loss(
            i, t.to(i.options()).to(torch::kLong),
            F::NLLLossFuncOptions().weight(weight.to(i.options())).reduction(torch::kNone))c                  h    t        j                  dd      j                  d      j                         S Nr   r2   {Gz?r8   r   addr   rk   r,   r*   rA   z0nllloss_no_reduce_weights_test.<locals>.<lambda>  %    B+//599; r,   r   r>   r:   r.   c                 j    t        d   | j                  |       j                         fi  |       S rh  rj  rk  s     r*   rA   z0nllloss_no_reduce_weights_test.<locals>.<lambda>  1    y)!QYYq\->->-@NF1IN r,   Fr   r   r8   rn  ro  r   rp  ra  r   r   rw   rb   ro   r:   r.   s   @@@r*   nllloss_no_reduce_weights_testr  	  s    R))+//399;@@BCAZZ^FB ,#EG_ <(qFCOll$ $r,   c            
      X    t        t        j                  d      j                         j	                  d      j                         j                               t        j                  d      fd t        dt         fd      dd dd	 fd
dt        j                        S )Nr   r2   c                 .    j                  |       dddS )Nr|   rP   r.   r!   ru  r  r  s    r*   ro   z;nllloss_no_reduce_weights_ignore_index_test.<locals>.kwargs#  s     ..+& !# 	#r,   &NLLLoss_no_reduce_weights_ignore_indexc                     t        j                  | j                  |       j                         fi  | j                        S rj   )r~   r`  r   ra  datarb  s    r*   rA   z=nllloss_no_reduce_weights_ignore_index_test.<locals>.<lambda>*  s0    ajjAIIaL$5$5$7J6!&&>J r,   zF::nll_loss(
            i, t.to(i.options()).to(torch::kLong),
            F::NLLLossFuncOptions().weight(weight.to(i.options())).reduction(torch::kNone).ignore_index(2))c                  h    t        j                  dd      j                  d      j                         S r  r  rk   r,   r*   rA   z=nllloss_no_reduce_weights_ignore_index_test.<locals>.<lambda>.  r  r,   r   r  c                 j    t        d   | j                  |       j                         fi  |       S rh  rj  rk  s     r*   rA   z=nllloss_no_reduce_weights_ignore_index_test.<locals>.<lambda>0  r  r,   Fr   r  r  s   @@@r*   +nllloss_no_reduce_weights_ignore_index_testr    s    R))+//399;@@BCAZZ^F# 9#JLo <(qFCOll$ $r,   c            
          t        t        j                  d      j                         j	                  d      j                         j                               t        j                  d      fd t        dt         fd      dt        j                  ddt        j                        j                  d      j                         d	d
 fddt        j                        S )Nr   r2   c                 .    j                  |       dddS )Nr|   r7   r  r  r  s    r*   ro   z?nllloss_no_reduce_weights_ignore_index_neg_test.<locals>.kwargs:  s     ..+& "$ 	$r,   *NLLLoss_no_reduce_weights_ignore_index_negc                 x    t        j                  | j                  |       j                         fi  |       S rj   r_  rb  s    r*   rA   zAnllloss_no_reduce_weights_ignore_index_neg_test.<locals>.<lambda>A  r  r,   zF::nll_loss(
            i, t.to(i.options()).to(torch::kLong),
            F::NLLLossFuncOptions().weight(weight.to(i.options())).reduction(torch::kNone).ignore_index(-1))r   r  r   r  c                 j    t        d   | j                  |       j                         fi  |       S rh  rj  rk  s     r*   rA   zAnllloss_no_reduce_weights_ignore_index_neg_test.<locals>.<lambda>G  r  r,   F)r   r   r   inputr   rG   r   rJ   )r   r8   rn  ro  r   rp  ra  r   r   rw   rb   r  r   r  s   @@@r*   /nllloss_no_reduce_weights_ignore_index_neg_testr  6  s    R))+//399;@@BCAZZ^F$ =#EGp jjRu||488>BBD(qFCOll$ $r,   c            
          t        t        j                  ddd      j                  d      j	                         j                               ddi t        dt         fd      dd	 d
d fddt        j                        S )NrP   rS   rW   r!   r|   NLLLoss2d_no_reducec                 t    t        j                  | j                  |       j                         d         S r^  r_  rb  s    r*   rA   z*nllloss2d_no_reduce_test.<locals>.<lambda>S  rc  r,   rd  c                  N    t        j                  dddd      j                         S NrP   rW   rS   r   rk   r,   r*   rA   z*nllloss2d_no_reduce_test.<locals>.<lambda>V      Aq!Q/335 r,   r   r   c                 ^    t        d   | j                  |       j                         fi S N	NLLLossNdrj  rk  s     r*   rA   z*nllloss2d_no_reduce_test.<locals>.<lambda>X  -    {+Aqyy|/@/@/BMfM r,   Fr   	r   r8   r   r   rp  ra  r   rw   rb   rq  s   @@r*   nllloss2d_no_reduce_testr  M  s~    Aq!$((+11388:;A6"F&#WYg5(q1Nll$ $r,   c            
          t        t        j                  ddd      j                  d      j	                         j                               ddd t        dt         fd      d	d
 dd fddt        j                        S )NrP   rS   rW   r6   r|   rt   NLLLoss2d_no_reduce_ignore_indexc           	          t        j                  | j                  |       j                         t	        d         t        d               S rx  ry  rb  s    r*   rA   z7nllloss2d_no_reduce_ignore_index_test.<locals>.<lambda>d  r|  r,   F::nll_loss(
            i, t.to(i.options()).to(torch::kLong), F::NLLLossFuncOptions().ignore_index(1).reduction(torch::kNone))c                  N    t        j                  dddd      j                         S r  r   rk   r,   r*   rA   z7nllloss2d_no_reduce_ignore_index_test.<locals>.<lambda>h  r  r,   r   r   c                 ^    t        d   | j                  |       j                         fi S r  rj  rk  s     r*   rA   z7nllloss2d_no_reduce_ignore_index_test.<locals>.<lambda>j  r  r,   Fr   r  rq  s   @@r*   %nllloss2d_no_reduce_ignore_index_testr  ^  s    Aq!$((+11388:;A:;&)QF3#EFw5(q1Nll$ $r,   c            
      @    t        t        j                  ddd      j                  d      j	                         j                               t        j                  d      fd t        dt         fd      dd d	d
 fddt        j                        S )NrP   rS   rW   c                 ,    j                  |       ddS r   r  r  s    r*   ro   z0nllloss2d_no_reduce_weights_test.<locals>.kwargst  r  r,   NLLLoss2d_no_reduce_weightsc                 x    t        j                  | j                  |       j                         fi  |       S rj   r_  rb  s    r*   rA   z2nllloss2d_no_reduce_weights_test.<locals>.<lambda>z  r  r,   r  c                  N    t        j                  dddd      j                         S r  r   rk   r,   r*   rA   z2nllloss2d_no_reduce_weights_test.<locals>.<lambda>~  r  r,   r   r  c                 j    t        d   | j                  |       j                         fi  |       S r  rj  rk  s     r*   rA   z2nllloss2d_no_reduce_weights_test.<locals>.<lambda>  1    {+Aqyy|/@/@/BPfQiP r,   Fr   r  r  s   @@@r*    nllloss2d_no_reduce_weights_testr  p  s    Aq!$((+11388:;AZZ]FB .#EG_ 6(qFCQll$ $r,   c            
          t        t        j                  ddddd      j                  d      j	                         j                               ddi t        dt         fd      dd	 d
d fddt        j                        S )NrP   rS   rW   r!   r|   NLLLossNd_no_reducec                 t    t        j                  | j                  |       j                         d         S r^  r_  rb  s    r*   rA   z*nlllossNd_no_reduce_test.<locals>.<lambda>  rc  r,   rd  c                  R    t        j                  dddddd      j                         S r  r   rk   r,   r*   rA   z*nlllossNd_no_reduce_test.<locals>.<lambda>  "    Aq!Q1599; r,   r   r   c                 ^    t        d   | j                  |       j                         fi S r  rj  rk  s     r*   rA   z*nlllossNd_no_reduce_test.<locals>.<lambda>  r  r,   Fr   r  rq  s   @@r*   nlllossNd_no_reduce_testr    s    Aq!Q*..q1779>>@AA6"F&#WYg;(q1Nll$ $r,   c            
          t        t        j                  ddddd      j                  d      j	                         j                               ddd t        dt         fd      d	d
 dd fddt        j                        S )NrP   rS   rW   r6   r|   rt   NLLLossNd_no_reduce_ignore_indexc           	          t        j                  | j                  |       j                         t	        d         t        d               S rx  ry  rb  s    r*   rA   z7nlllossNd_no_reduce_ignore_index_test.<locals>.<lambda>  r|  r,   r  c                  R    t        j                  dddddd      j                         S r  r   rk   r,   r*   rA   z7nlllossNd_no_reduce_ignore_index_test.<locals>.<lambda>  r  r,   r   r   c                 ^    t        d   | j                  |       j                         fi S r  rj  rk  s     r*   rA   z7nlllossNd_no_reduce_ignore_index_test.<locals>.<lambda>  r  r,   Fr   r  rq  s   @@r*   %nlllossNd_no_reduce_ignore_index_testr    s    Aq!Q*..q1779>>@AA:;&)QF3#EFw;(q1Nll$ $r,   c            
      D    t        t        j                  ddddd      j                  d      j	                         j                               t        j                  d      fd t        dt         fd      dd d	d
 fddt        j                        S )NrP   rS   rW   c                 ,    j                  |       ddS r   r  r  s    r*   ro   z0nlllossNd_no_reduce_weights_test.<locals>.kwargs  r  r,   NLLLossNd_no_reduce_weightsc                 x    t        j                  | j                  |       j                         fi  |       S rj   r_  rb  s    r*   rA   z2nlllossNd_no_reduce_weights_test.<locals>.<lambda>  r  r,   r  c                  R    t        j                  dddddd      j                         S r  r   rk   r,   r*   rA   z2nlllossNd_no_reduce_weights_test.<locals>.<lambda>  r  r,   r   r  c                 j    t        d   | j                  |       j                         fi  |       S r  rj  rk  s     r*   rA   z2nlllossNd_no_reduce_weights_test.<locals>.<lambda>  r  r,   Fr   r  r  s   @@@r*    nlllossNd_no_reduce_weights_testr    s    Aq!Q*..q1779>>@AAZZ]FB .#EG_ <(qFCQll$ $r,   c                       t        j                  dddt         j                         t        dt	         fd      dd d	 d
 fdddt         j                  	      S )NrP   rW   r4   r   SmoothL1Loss_no_reducec                 R    t        j                  | j                  |       d      S r{   r~   smooth_l1_lossr   r   s    r*   rA   z-smoothl1loss_no_reduce_test.<locals>.<lambda>      a&&q!))A,&I r,   jF::smooth_l1_loss(
            i, t.to(i.options()), F::SmoothL1LossFuncOptions().reduction(torch::kNone))c                  0    t        j                  ddd      S r-  r.  rk   r,   r*   rA   z-smoothl1loss_no_reduce_test.<locals>.<lambda>  r/  r,   r   r   c                 D    t        d   | j                  |       d      S NSmoothL1Lossr|   r}   r   r   s     r*   rA   z-smoothl1loss_no_reduce_test.<locals>.<lambda>  !    ~.q!))A,&Q r,   TFr   r4  r   s   @r*   smoothl1loss_no_reduce_testr    s[    Aq!5<<0A)#IK[-(q1R ll$ $r,   c                       t        j                  dt         j                         t        dt	         fd      dd d d fd	d
dt         j                  	      S )Nrk   r   SmoothL1Loss_no_reduce_scalarc                 R    t        j                  | j                  |       d      S r{   r  r   s    r*   rA   z4smoothl1loss_no_reduce_scalar_test.<locals>.<lambda>  r  r,   r  c                  ,    t        j                  d      S r  r.  rk   r,   r*   rA   z4smoothl1loss_no_reduce_scalar_test.<locals>.<lambda>  rA  r,   r   r   c                 D    t        d   | j                  |       d      S r  r   r   s     r*   rA   z4smoothl1loss_no_reduce_scalar_test.<locals>.<lambda>  r  r,   TFr   r4  r   s   @r*   "smoothl1loss_no_reduce_scalar_testr    sW    Bell+A0#IK[((q1R ll$ $r,   c                       t        j                  dddt         j                         t        dt	         fd      dd d	 d
 fdddt         j                  	      S )NrP   rW   r4   r   SmoothL1Loss_betac                 T    t        j                  | j                  |       dd      S )Nr|         ?r!   betar  r   s    r*   rA   z(smoothl1loss_beta_test.<locals>.<lambda>  s     a&&q!))A,&sS r,   zoF::smooth_l1_loss(
            i, t.to(i.options()), F::SmoothL1LossFuncOptions().reduction(torch::kNone), 0.5)c                  0    t        j                  ddd      S r-  r.  rk   r,   r*   rA   z(smoothl1loss_beta_test.<locals>.<lambda>  r/  r,   r   r   c                 F    t        d   | j                  |       dd      S )Nr  r|   r  r  r   r   s     r*   rA   z(smoothl1loss_beta_test.<locals>.<lambda>  s$    ~.q!))A,&WZ[ r,   TFr   r4  r   s   @r*   smoothl1loss_beta_testr    s[    Aq!5<<0A$#SU`-(q1\ ll$ $r,   c                       t        j                  dddt         j                         t        dt	         fd      dd d	 d
 fdddt         j                  	      S )NrP   rW   r4   r   SmoothL1Loss_zero_betac                 T    t        j                  | j                  |       dd      S )Nr|   r   r  r  r   s    r*   rA   z-smoothl1loss_zero_beta_test.<locals>.<lambda>  s     a&&q!))A,&qQ r,   zmF::smooth_l1_loss(
            i, t.to(i.options()), F::SmoothL1LossFuncOptions().reduction(torch::kNone), 0)c                  0    t        j                  ddd      S r-  r.  rk   r,   r*   rA   z-smoothl1loss_zero_beta_test.<locals>.<lambda>  r/  r,   r   r   c                 F    t        d   | j                  |       dd      S )Nr  r|   r   r  r   r   s     r*   rA   z-smoothl1loss_zero_beta_test.<locals>.<lambda>  s$    ~.q!))A,&WXY r,   TFr   r4  r   s   @r*   smoothl1loss_zero_beta_testr    s[    Aq!5<<0A)#QS^-(q1Z ll$ $r,   c                       t        j                  ddd       t        dt         fd      dd d d	 fd
ddt         j                  	      S )NrP   rW   r4   HuberLoss_deltac                 T    t        j                  | j                  |       dd      S )Nr|   r  r!   delta)r~   
huber_lossr   r   s    r*   rA   z&huberloss_delta_test.<locals>.<lambda>  s    all1aiilfCP r,   znF::huber_loss(
            i, t.to(i.options()), F::HuberLossFuncOptions().reduction(torch::kNone).delta(0.5))c                  0    t        j                  ddd      S r-  r.  rk   r,   r*   rA   z&huberloss_delta_test.<locals>.<lambda>  r/  r,   r   r   c                 F    t        d   | j                  |       dd      S )N	HuberLossr|   r  r  r   r   s     r*   rA   z&huberloss_delta_test.<locals>.<lambda>  s$    {+Aqyy|vUXY r,   TFr   r   r   s   @r*   huberloss_delta_testr    sU    Aq!A"#PRc-(q1Z ll$ $r,   c                       t        j                  d      j                          t        dt	         fd      dd d d fdd	d
d
	      S )Nrk   !MultiLabelMarginLoss_0d_no_reducec                 n    t        j                  | j                  |       j                         d      S r{   r~   multilabel_margin_lossr   ra  r   s    r*   rA   z8multilabelmarginloss_0d_no_reduce_test.<locals>.<lambda>  (    a..q!))A,2C2C2EQWX r,   F::multilabel_margin_loss(
            i, t.to(i.options()).to(torch::kLong), F::MultilabelMarginLossFuncOptions().reduction(torch::kNone))c                  ,    t        j                  d      S r  r.  rk   r,   r*   rA   z8multilabelmarginloss_0d_no_reduce_test.<locals>.<lambda>  rA  r,   r   r   c                 t    t        d   | j                  j                  |       j                         d      S NMultiLabelMarginLossr|   r}   r   r  r   ra  r   s     r*   rA   z8multilabelmarginloss_0d_no_reduce_test.<locals>.<lambda>  1    56q!&&..:K:P:P:R^de r,   TF)	r   r   r   r   r   rG   check_sum_reductionr^   r   )r8   zerosra  r   rw   r   s   @r*   &multilabelmarginloss_0d_no_reduce_testr    sU    BA4#XZt((q1f  r,   c                      t        t        j                  d      j                  d      j	                         j                                t        dt         fd      dd d d fdd	d
d
t        j                  
      S )Nr2   !MultiLabelMarginLoss_1d_no_reducec                 n    t        j                  | j                  |       j                         d      S r{   r  r   s    r*   rA   z8multilabelmarginloss_1d_no_reduce_test.<locals>.<lambda>*  r  r,   r  c                  ,    t        j                  d      S r   r.  rk   r,   r*   rA   z8multilabelmarginloss_1d_no_reduce_test.<locals>.<lambda>-  rA  r,   r   r   c                 t    t        d   | j                  j                  |       j                         d      S r	  r  r   s     r*   rA   z8multilabelmarginloss_1d_no_reduce_test.<locals>.<lambda>/  r  r,   TF
r   r   r   r   r   rG   r  r^   r   rJ   r  r   s   @r*   &multilabelmarginloss_1d_no_reduce_testr  %  su    B##B'--/4467A4#XZt((q1f ll$ $r,   c                  L    t        t        j                  t        j                  dd      j	                  d      j                  d      j                         j                         d             t        dt         fd      d	d
 d d fddddt        j                  
      S )NrS   r2   g         r7   minMultiLabelMarginLoss_index_negc                 n    t        j                  | j                  |       j                         d      S r{   r  r   s    r*   rA   z5multilabelmarginloss_index_neg_test.<locals>.<lambda><  r  r,   r  c                  .    t        j                  dd      S NrS   r2   r.  rk   r,   r*   rA   z5multilabelmarginloss_index_neg_test.<locals>.<lambda>?      Q+ r,   r   r   c                 t    t        d   | j                  j                  |       j                         d      S r	  r  r   s     r*   rA   z5multilabelmarginloss_index_neg_test.<locals>.<lambda>A  r  r,   TFr  )r   r8   clampr   r  r   rp  ra  r   rw   rb   r   s   @r*   #multilabelmarginloss_index_neg_testr"  7  s    UZZ2.2237;;B?EEGLLNTVWXA1#XZt+(q1f ll$ $r,   c                      t        t        j                  dd      j                  d      j	                         j                                t        dt         fd      dd d d fd	d
ddt        j                  
      S )NrS   r2   MultiLabelMarginLoss_no_reducec                 n    t        j                  | j                  |       j                         d      S r{   r  r   s    r*   rA   z5multilabelmarginloss_no_reduce_test.<locals>.<lambda>N  r  r,   r  c                  .    t        j                  dd      S r  r.  rk   r,   r*   rA   z5multilabelmarginloss_no_reduce_test.<locals>.<lambda>Q  r  r,   r   r   c                 t    t        d   | j                  j                  |       j                         d      S r	  r  r   s     r*   rA   z5multilabelmarginloss_no_reduce_test.<locals>.<lambda>S  r  r,   TFr  r  r   s   @r*   #multilabelmarginloss_no_reduce_testr(  I  sx    Ar"&&r*002779:A1#XZt+(q1f ll$ $r,   c                  >    t        t        j                  d      j                  d      j	                  t        j
                        j                  d      j                  d             t        dt         fd      dd d	 d
 fdddt        j
                  	      S )Nr2   r   rP   r6   HingeEmbeddingLoss_no_reducec                 R    t        j                  | j                  |       d      S r{   r~   hinge_embedding_lossr   r   s    r*   rA   z3hingeembeddingloss_no_reduce_test.<locals>.<lambda>`  r   r,   zvF::hinge_embedding_loss(
            i, t.to(i.options()), F::HingeEmbeddingLossFuncOptions().reduction(torch::kNone))c                  ,    t        j                  d      S r   r.  rk   r,   r*   rA   z3hingeembeddingloss_no_reduce_test.<locals>.<lambda>c  rA  r,   r   r   c                 D    t        d   | j                  |       d      S )NHingeEmbeddingLossr|   r}   r   r   s     r*   rA   z3hingeembeddingloss_no_reduce_test.<locals>.<lambda>e  s#    34Q		!PVW r,   TF	r   r   r   r   r   rG   r  r   rJ   
r   r8   r   r   r   rb   mul_subr   rw   r   s   @r*   !hingeembeddingloss_no_reduce_testr5  [  s    R##A&))%,,7<<Q?CCAFGA/#OQa((q1X ll$ $r,   c                  >    t        t        j                  d      j                  d      j	                  t        j
                        j                  d      j                  d             t        dt         fd      dd d	 d
 fdddt        j
                  	      S )Nr2   r   rP   r6   #HingeEmbeddingLoss_margin_no_reducec                 T    t        j                  | j                  |       dd      S Nr  r|   marginr!   r,  r   s    r*   rA   z:hingeembeddingloss_margin_no_reduce_test.<locals>.<lambda>q  s!    a,,Q		!STZ[ r,   zF::hinge_embedding_loss(
            i, t.to(i.options()), F::HingeEmbeddingLossFuncOptions().margin(0.5).reduction(torch::kNone))c                  ,    t        j                  d      S r   r.  rk   r,   r*   rA   z:hingeembeddingloss_margin_no_reduce_test.<locals>.<lambda>t  rA  r,   r   r   c                 F    t        d   | j                  |       dd      S )Nr0  r  r|   r:  r   r   s     r*   rA   z:hingeembeddingloss_margin_no_reduce_test.<locals>.<lambda>v  s%    34Q		!S\bc r,   TFr1  r2  r   s   @r*   (hingeembeddingloss_margin_no_reduce_testr>  l  s    R##A&))%,,7<<Q?CCAFGA6#[]m((q1d ll$ $r,   c                       t        j                  ddt         j                         t        dt	         fd      dd d d fd	d
dt         j                  	      S )NrS   r   SoftMarginLoss_no_reducec                 R    t        j                  | j                  |       d      S r{   )r~   soft_margin_lossr   r   s    r*   rA   z/softmarginloss_no_reduce_test.<locals>.<lambda>  r   r,   znF::soft_margin_loss(
            i, t.to(i.options()), F::SoftMarginLossFuncOptions().reduction(torch::kNone))c                  .    t        j                  dd      S )NrS   r.  rk   r,   r*   rA   z/softmarginloss_no_reduce_test.<locals>.<lambda>  s    Q* r,   r   r   c                 D    t        d   | j                  |       d      S )NSoftMarginLossr|   r}   r   r   s     r*   rA   z/softmarginloss_no_reduce_test.<locals>.<lambda>  s"    /0AIIaLFS r,   TFr   r4  r   s   @r*   softmarginloss_no_reduce_testrF  }  sY    Aq-A+#KM]*(q1T ll$ $r,   c                       t        j                  dd      j                  d      j                          t	        dt         fd      dd d d	 fd
ddt         j                  	      S )NrS   r2   rP   "MultiLabelSoftMarginLoss_no_reducec                 R    t        j                  | j                  |       d      S r{   r~   multilabel_soft_margin_lossr   r   s    r*   rA   z9multilabelsoftmarginloss_no_reduce_test.<locals>.<lambda>  s    a33Aqyy|vV r,   zF::multilabel_soft_margin_loss(
            i, t.to(i.options()), F::MultilabelSoftMarginLossFuncOptions().reduction(torch::kNone))c                  .    t        j                  dd      S r  r.  rk   r,   r*   rA   z9multilabelsoftmarginloss_no_reduce_test.<locals>.<lambda>  r  r,   r   r   c                     | j                         j                         z  dz
  |  j                         j                         z  z    j                  d      | j                  d      z  S Nr6   dimr   r   sumrc   r   s     r*   rA   z9multilabelsoftmarginloss_no_reduce_test.<locals>.<lambda>  s`    199;??$$A1"1C1C1E'EEFKKPQKRUVU[U[\]U^^ r,   Fr   r8   r   r   rp  r   rw   rb   r   s   @r*   'multilabelsoftmarginloss_no_reduce_testrT    sg    

1ba &&(A5#VXg+(q1_ll$ $r,   c                      t        j                  dd      j                  d      j                          t        j                  d      t	        dt         fd      dd d d	 fd
dddt         j                  
      S )NrS   r2   rP   *MultiLabelSoftMarginLoss_weights_no_reducec                 r    t        j                  | j                  |       j                  |       d      S r   rJ  r   s    r*   rA   zAmultilabelsoftmarginloss_weights_no_reduce_test.<locals>.<lambda>  s.    a33Aqyy|;B??1;MY_a r,   zF::multilabel_soft_margin_loss(
            i, t.to(i.options()),
            F::MultilabelSoftMarginLossFuncOptions().weight(weights.to(i.options())).reduction(torch::kNone))c                  .    t        j                  dd      S r  r.  rk   r,   r*   rA   zAmultilabelsoftmarginloss_weights_no_reduce_test.<locals>.<lambda>  r  r,   r   r   c                     | j                         j                         z  dz
  |  j                         j                         z  z    z  j                  d      | j                  d      z  S rN  rQ  r   s     r*   rA   zAmultilabelsoftmarginloss_weights_no_reduce_test.<locals>.<lambda>  se    199;??$$A1"1C1C1E'EEFPUUZ[U\_`_e_efg_hh r,   TFr  rS  r   s   @@r*   /multilabelsoftmarginloss_weights_no_reduce_testrZ    s~    

1ba &&(AjjnG=#abq ,(qWEi ll$ $r,   c                       t        j                  d      j                  d      j                         j	                          t        dt         fd      dd d d fd	d
ddt         j                  
      S )NrS   r3   MultiMarginLoss_no_reducec                 n    t        j                  | j                  |       j                         d      S r{   r~   multi_margin_lossr   ra  r   s    r*   rA   z0multimarginloss_no_reduce_test.<locals>.<lambda>  '    a))!QYYq\->->-@FS r,   F::multi_margin_loss(
            i, t.to(i.options()).to(torch::kLong), F::MultiMarginLossFuncOptions().reduction(torch::kNone))c                  .    t        j                  dd      S r  r.  rk   r,   r*   rA   z0multimarginloss_no_reduce_test.<locals>.<lambda>  r  r,   r   r   c                 t    t        d   | j                  j                  |       j                         d      S NMultiMarginLossr|   r}   r  r   s     r*   rA   z0multimarginloss_no_reduce_test.<locals>.<lambda>  1    01!QVV^^A5F5K5K5MY_` r,   TFr  r8   r   r   rp  ra  r   rw   rb   r   s   @r*   multimarginloss_no_reduce_testrh    sp    

1!""$))+A,#SUo+(q1a ll$ $r,   c                       t        j                  d      j                  d      j                         j	                          t        dt         fd      dd d d fd	d
ddt         j                  
      S )Nr6   r3   MultiMarginLoss_1d_no_reducec                 n    t        j                  | j                  |       j                         d      S r{   r^  r   s    r*   rA   z3multimarginloss_1d_no_reduce_test.<locals>.<lambda>  r`  r,   ra  c                  ,    t        j                  d      S r   r.  rk   r,   r*   rA   z3multimarginloss_1d_no_reduce_test.<locals>.<lambda>  rA  r,   r   r   c                 t    t        d   | j                  j                  |       j                         d      S rd  r  r   s     r*   rA   z3multimarginloss_1d_no_reduce_test.<locals>.<lambda>  rf  r,   TFr  rg  r   s   @r*   !multimarginloss_1d_no_reduce_testrn    sp    

1!""$))+A/#SUo((q1a ll$ $r,   c                       t        j                  d      j                  d      j                         j	                          t        dt         fd      dd d d fd	d
ddt         j                  
      S )Nrk   r3   ,multimarginloss_1d_input_0d_target_no_reducec                 n    t        j                  | j                  |       j                         d      S r{   r^  r   s    r*   rA   zCmultimarginloss_1d_input_0d_target_no_reduce_test.<locals>.<lambda>  r`  r,   ra  c                  ,    t        j                  d      S r   r.  rk   r,   r*   rA   zCmultimarginloss_1d_input_0d_target_no_reduce_test.<locals>.<lambda>  rA  r,   r   r   c                 t    t        d   | j                  j                  |       j                         d      S rd  r  r   s     r*   rA   zCmultimarginloss_1d_input_0d_target_no_reduce_test.<locals>.<lambda>  rf  r,   TFr  rg  r   s   @r*   1multimarginloss_1d_input_0d_target_no_reduce_testrt    sp    

21##%**,A?#SUo((q1a ll$ $r,   c                       t        j                  d      j                  d      j                         j	                          t        dt         fd      dd d d fd	d
ddt         j                  
      S )NrS   r3   MultiMarginLoss_p_no_reducec                 p    t        j                  | j                  |       j                         dd      S )NrP   r|   r?   r!   r^  r   s    r*   rA   z2multimarginloss_p_no_reduce_test.<locals>.<lambda>  s*    a))!QYYq\->->-@AQWX r,   zF::multi_margin_loss(
            i, t.to(i.options()).to(torch::kLong), F::MultiMarginLossFuncOptions().p(2).reduction(torch::kNone))c                  N    t        j                  dd      j                  dd      S )NrS   r2   r  gGz?)r8   r   r   rk   r,   r*   rA   z2multimarginloss_p_no_reduce_test.<locals>.<lambda>  s    Q+224B r,   r   r   c                 v    t        d   | j                  j                  |       j                         dd      S )Nre  rP   r|   rx  r  r   s     r*   rA   z2multimarginloss_p_no_reduce_test.<locals>.<lambda>  s4    01!QVV^^A5F5K5K5MQR^de r,   TFr  rg  r   s   @r*    multimarginloss_p_no_reduce_testr{    sp    

1!""$))+A.#XZtB(q1f ll$ $r,   c                       t        j                  d      j                  d      j                         j	                          t        dt         fd      dd d d fd	d
ddt         j                  
      S )NrS   r3    MultiMarginLoss_margin_no_reducec                 p    t        j                  | j                  |       j                         dd      S r9  r^  r   s    r*   rA   z7multimarginloss_margin_no_reduce_test.<locals>.<lambda>  s*    a))!QYYq\->->-@X^_ r,   zF::multi_margin_loss(
            i, t.to(i.options()).to(torch::kLong),
            F::MultiMarginLossFuncOptions().margin(0.5).reduction(torch::kNone))c                  .    t        j                  dd      S r  r.  rk   r,   r*   rA   z7multimarginloss_margin_no_reduce_test.<locals>.<lambda>  r  r,   r   r   c                 v    t        d   | j                  j                  |       j                         dd      S )Nre  r  r|   r:  r  r   s     r*   rA   z7multimarginloss_margin_no_reduce_test.<locals>.<lambda>  s6    01!QVV^^A5F5K5K5M9<P r,   TFr  rg  r   s   @r*   %multimarginloss_margin_no_reduce_testr    st    

1!""$))+A3#_aT ,(q1P !ll$ $r,   c                  B    t        j                  d      j                  d      j                         j	                          t        j                  dt         j
                        t        dt         fd      dd d	 d
 fddddt         j
                  
      S )NrS   r3   r2   r   !MultiMarginLoss_weights_no_reducec                     t        j                  | j                  |       j                         j                  |       d      S r   r^  r   s    r*   rA   z8multimarginloss_weights_no_reduce_test.<locals>.<lambda>  s6    a))!QYYq\->->-@YZI[4:< r,   zF::multi_margin_loss(
            i, t.to(i.options()).to(torch::kLong),
            F::MultiMarginLossFuncOptions().weight(weights.to(i.options())).reduction(torch::kNone))c                  .    t        j                  dd      S r  r.  rk   r,   r*   rA   z8multimarginloss_weights_no_reduce_test.<locals>.<lambda>  r  r,   r   r   c                 v    t        d   | j                  j                  |       j                         d      S )Nre  r|   r   r  r   s     r*   rA   z8multimarginloss_weights_no_reduce_test.<locals>.<lambda>  s6    01!QVV^^A5F5K5K5M9@FT r,   TFr  )r8   r   r   rp  ra  rb   r   rw   r   s   @@r*   &multimarginloss_weights_no_reduce_testr    s    

1!""$))+Ajj5<<0G4#<=h ,(qWET !ll!$ $r,   c                     d } ||       }t        |t        j                        r|gn|}t               5   || j	                  d      cddd       S # 1 sw Y   yxY w)zReference function for modules supporting no batch dimensions.

    The module is passed the input and target in batched form with a single item.
    The output is squeezed to compare with the no-batch input.
    c                     t        | t        t        f      r| D cg c]  }|j                  d       c}S | j                  d      S c c}w rM   
isinstancelisttuple	unsqueezeinpr:   s     r*   unsqueeze_inpz0single_batch_reference_fn.<locals>.unsqueeze_inp,  =    cD%=),/0qAKKN00}}Q 1   Ar   N)r  r8   Tensorr   squeeze)r  
parametersmoduler  single_batch_inputs        r*   single_batch_reference_fnr  &  sa     
 'u-1;<NPUP\P\1],-cu		 6)*22156 6 6s   AAConv1d)r4   rS   rW   z!torch::nn::Conv1dOptions(4, 5, 3))rP   r4   r2   )rC   rD   rE   rF   cudnnrH   rI   rJ   )r4   rS   rW   rP   z+torch::nn::Conv1dOptions(4, 5, 3).stride(2)stride)	rC   rD   rE   rF   r  rN   rH   rI   rJ   )r4   rS   rW   r6   r6   z6torch::nn::Conv1dOptions(4, 5, 3).stride(1).padding(1)pad1r  )r4   rS   rS   r6   rP   z6torch::nn::Conv1dOptions(4, 5, 5).stride(1).padding(2)pad2)r4   r4   rW   r6   r6   z6torch::nn::Conv1dOptions(4, 4, 3).stride(1).padding(1))r6   r4   r6   	pad1size1)r4   r4   rS   r6   rP   z6torch::nn::Conv1dOptions(4, 4, 5).stride(1).padding(2)	pad2size1)r   r4   r2   
zero_batch)rC   rD   rE   rF   r  rN   rH   rI   Conv1d_dilatedc                  4    t        j                  dddd      S )Nr4   rS   rW   rP   kernel_sizedilationru   r  rk   r,   r*   rA   rA         BIIaAF r,   z-torch::nn::Conv1dOptions(4, 5, 3).dilation(2))r   r   rE   rF   rH   rI   rJ   Conv1d_groupsc                  4    t        j                  dddd      S )Nr4   r]   rW   rP   r  groupsr  rk   r,   r*   rA   rA         BIIa!D r,   z+torch::nn::Conv1dOptions(4, 6, 3).groups(2))rP   r4   r]   )r   r   rE   rF   r  rH   rI   rJ   Conv1d_pad_validc                  4    t        j                  dddd      S )Nr4   rS   rW   validpaddingr  rk   r,   r*   rA   rA     s    BIIaAw? r,   z8torch::nn::Conv1dOptions(4, 5, 3).padding(torch::kValid)Conv1d_pad_samec                  4    t        j                  dddd      S )Nr4   rS   rW   samer  r  rk   r,   r*   rA   rA         BIIaAv> r,   z7torch::nn::Conv1dOptions(4, 5, 3).padding(torch::kSame)Conv1d_pad_same2c                  4    t        j                  dddd      S )Nr4   rS   r  r  r  rk   r,   r*   rA   rA     r  r,   z7torch::nn::Conv1dOptions(4, 5, 4).padding(torch::kSame)Conv1d_pad_same_dilatedc                  6    t        j                  ddddd      S )Nr4   rS   r  rP   r  r  r  rk   r,   r*   rA   rA     s    BIIaAvJ r,   zCtorch::nn::Conv1dOptions(4, 5, 3).padding(torch::kSame).dilation(2)ConvTranspose1dc                  8    t        j                  dddddd      S )NrW   r4   rW   r6   r6   )r  r  r  output_paddingru   r  rk   r,   r*   rA   rA     s    B..q!4YZkop r,   zQtorch::nn::ConvTranspose1dOptions(3, 4, 3).stride(3).padding(1).output_padding(1))r6   rW      )r   r   rE   r  rF   rH   rI   rJ   )rW   r4   rW   rP   r6   r6   r6   Fztorch::nn::ConvTranspose1dOptions(3, 4, 3)
                                .stride(2).padding(1).output_padding(1).groups(1).bias(false))r6   rW   r]   )	rW   r4   rW   rP   r6   r6   r6   TrP   ztorch::nn::ConvTranspose1dOptions(3, 4, 3)
                                .stride(2).padding(1).output_padding(1).groups(1).bias(true).dilation(2)dilatedConvTranspose1d_groupsc            	      :    t        j                  ddddddd      S )	Nr4   r]   rW   r  r6   r  rP   )r  r  r  r  r  rk   r,   r*   rA   rA     s    B..q!QtQ_clmn r,   z|torch::nn::ConvTranspose1dOptions(4, 6, 3)
                                .stride(3).padding(1).output_padding(1).groups(2))rP   r4   r  Conv2d)rW   r4   rW   rP   z&torch::nn::Conv2dOptions(3, 4, {3, 2}))rP   rW   r  rS   )	rC   rD   rE   rF   r  check_with_long_tensorrH   rI   rJ   )rW   r4   rW   rW   rP   rP   z5torch::nn::Conv2dOptions(3, 4, {3, 3}).stride({2, 2})strided)
rC   rD   rE   rF   r  rN   r  rH   rI   rJ   )rW   r4   r  r  r6   r6   zEtorch::nn::Conv2dOptions(3, 4, {3, 3}).stride({2, 2}).padding({1, 1})r  )rW   rP   r  r  r  r  zVtorch::nn::Conv2dOptions(3, 2, {3, 3}).stride({2, 2}).padding({1, 1}).dilation({2, 2}))rP   rW   r3   r3   )rW   r4   r  r6   r   r6   r6   Fz~torch::nn::Conv2dOptions(3, 4, {3, 2})
                                .stride(1).padding(0).dilation(1).groups(1).bias(false))rP   rW   r]   rS   gQ?)r   rW   r  rS   )rC   rD   rE   rF   r  rN   r  rH   Conv2d_groupsc                  4    t        j                  dddd      S Nr4   r]   r  rP   r  ru   r  rk   r,   r*   rA   rA   e      BIIaF1= r,   z0torch::nn::Conv2dOptions(4, 6, {3, 2}).groups(2))rP   r4   r]   rS   )	r   r   rE   rF   r  r  rH   rI   rJ   Conv2d_groups_thnnc                  4    t        j                  dddd      S r  r  rk   r,   r*   rA   rA   p  r  r,   )r   r   rE   rF   r  rH   rI   rJ   Conv2d_pad_validc                  4    t        j                  dddd      S )NrP   r4   rW   r4   r  r  r  rk   r,   r*   rA   rA   z  s    BIIaFGD r,   z=torch::nn::Conv2dOptions(2, 4, {3, 4}).padding(torch::kValid))rP   rP   r]   rS   Conv2d_pad_samec                  4    t        j                  dddd      S )NrP   r4   r  r  r  r  rk   r,   r*   rA   rA     s    BIIaFFC r,   z<torch::nn::Conv2dOptions(2, 4, {3, 4}).padding(torch::kSame)Conv2d_pad_same_dilatedc                  6    t        j                  ddddd      S )NrP   r4   r  r  r  r  rk   r,   r*   rA   rA     s    BIIaFFQO r,   zHtorch::nn::Conv2dOptions(2, 4, {3, 4}).padding(torch::kSame).dilation(2)ConvTranspose2d)rW   r4   rW   r  r6   r  z|torch::nn::ConvTranspose2dOptions(3, 4, 3)
                                .stride({3, 2}).padding(1).output_padding({1, 1}))r6   rW   r  r]   )	rC   rD   rE   r  rF   r  rH   rI   rJ   )	rW   r4   rW   rP   rW   r6   r  r6   Fr  aH  torch::nn::ConvTranspose2dOptions(3, 4, 3)
                                .stride({2, 3})
                                .padding(1)
                                .output_padding({1, 1})
                                .groups(1)
                                .bias(false)
                                .dilation({2, 2}))r6   rW   r]   r  )rW   r4   rW   r  r6   r  r6   Fztorch::nn::ConvTranspose2dOptions(3, 4, 3)
                                .stride({2, 3}).padding(1).output_padding({1, 1}).groups(1).bias(false)ConvTranspose2d_groupsc                  4    t        j                  dddd      S )NrP   r4   r  r  )ru   r  rk   r,   r*   rA   rA     s    B..q!VAF r,   z9torch::nn::ConvTranspose2dOptions(2, 4, {2, 3}).groups(2))r6   rP   r4   rS   Conv2d_depthwisec                  4    t        j                  dddd      S )Nr4   r  r  r  rk   r,   r*   rA   rA     r  r,   z0torch::nn::Conv2dOptions(4, 4, {3, 3}).groups(4))rP   r4   r]   r]    Conv2d_depthwise_with_multiplierc                  4    t        j                  dddd      S )Nr4   r3   r  r  r  rk   r,   r*   rA   rA     r  r,   z0torch::nn::Conv2dOptions(4, 8, {3, 3}).groups(4)Conv2d_depthwise_stridedc                  6    t        j                  ddddd      S )Nr4   r  r  )r  r  r  rk   r,   r*   rA   rA     s    BIIaF6!L r,   z?torch::nn::Conv2dOptions(4, 4, {3, 3}).stride({2, 2}).groups(4)Conv2d_depthwise_paddedc                  6    t        j                  ddddd      S )Nr4   r  r  )r  r  r  rk   r,   r*   rA   rA     s    BIIaFF1M r,   z@torch::nn::Conv2dOptions(4, 4, {3, 3}).padding({1, 1}).groups(4)Conv2d_depthwise_dilatedc                  6    t        j                  ddddd      S )Nr4   r  )r  r  r  rk   r,   r*   rA   rA     s    BIIaFVAN r,   zAtorch::nn::Conv2dOptions(4, 4, {2, 2}).dilation({2, 2}).groups(4))rP   r4   rS   rS   Conv3d)rP   rW   rP   rW   rP   z)torch::nn::Conv3dOptions(2, 3, {2, 3, 2}))r6   rP   r4   rS   r4   g?)rP   rW   rP   rW   r4   r6   r   r6   r6   Fztorch::nn::Conv3dOptions(2, 3, {2, 3, 4})
                                .stride(1).padding(0).dilation(1).groups(1).bias(false))r6   rP   rW   r4   rS   )rP   rW   )r6   r6   r6   r6   r   r6   r6   F1x1x1_no_bias)rW   r4   rP   rP   z+torch::nn::Conv3dOptions(3, 4, 2).stride(2))rP   rW   rS   rS   rS   )rW   r4   rP   rP   r6   z6torch::nn::Conv3dOptions(3, 4, 2).stride(2).padding(1)stride_padding)rW   r4   r  z)torch::nn::Conv3dOptions(3, 4, {2, 3, 4}))r   rW   rW   r4   rS   )rC   rD   rE   rF   r  r  rN   rH   Conv3d_groupsc                  4    t        j                  dddd      S )NrP   r4   rW   r  ru   r  rk   r,   r*   rA   rA   B  r  r,   z+torch::nn::Conv3dOptions(2, 4, 3).groups(2)Conv3d_dilatedc                  4    t        j                  dddd      S )NrW   r4   rP   r  r  rk   r,   r*   rA   rA   M  r  r,   z-torch::nn::Conv3dOptions(3, 4, 2).dilation(2)Conv3d_dilated_stridedc                  6    t        j                  ddddd      S )NrW   r4   rP   )r  r  r  r  rk   r,   r*   rA   rA   V  s    BIIaAaP r,   z7torch::nn::Conv3dOptions(3, 4, 2).dilation(2).stride(2)Conv3d_pad_validc                  4    t        j                  dddd      S )NrW   r4   r  r  r  r  rk   r,   r*   rA   rA   _  s    BIIaIwG r,   z@torch::nn::Conv3dOptions(3, 4, {2, 3, 4}).padding(torch::kValid))rP   rW   r]   rS   r4   Conv3d_pad_samec                  4    t        j                  dddd      S )NrW   r4   r  r  r  r  rk   r,   r*   rA   rA   i  s    BIIaIvF r,   z?torch::nn::Conv3dOptions(3, 4, {2, 3, 4}).padding(torch::kSame)Conv3d_pad_same_dilatedc                  6    t        j                  ddddd      S )NrW   r4   r  r  rP   r  r  rk   r,   r*   rA   rA   s  s    BIIaIvPQR r,   zKtorch::nn::Conv3dOptions(3, 4, {2, 3, 4}).padding(torch::kSame).dilation(2)ConvTranspose3dz2torch::nn::ConvTranspose3dOptions(2, 3, {2, 3, 2}))rC   rD   rE   r  rF   rH   rI   rJ   )	rP   rW   r  r6   r   r   r6   T)rP   rP   rP   ztorch::nn::ConvTranspose3dOptions(2, 3, {2, 3, 2})
                                .stride(1).padding(0).output_padding(0).groups(1).bias(true).dilation({2, 2, 2}))	rC   rD   rE   r  rF   rN   rH   rI   rJ   ReplicationPad3d))r6   rP   rW   rW   rP   r6   z6torch::nn::ReplicationPad3dOptions({1, 2, 3, 3, 2, 1}))rP   rW   rP   rP   rP   )rC   rD   rE   rF   rJ   )rW   rP   rP   rP   no_batch_dim)rC   rD   rE   rF   rG   rN   rJ   c            	      V    t        j                  dddddt         j                  d      S )NrP   rW   T)r   requires_grad)r8   r   
complex128rk   r,   r*   rA   rA     s"    Aq!Q9I9IY]^ r,   complex)rC   rD   rE   r   	skip_halfrN   	Embedding)r4   rW   z!torch::nn::EmbeddingOptions(4, 3)c                  l    t        j                  ddt         j                        j                  d      S NrP   rW   r   r4   r8   rn  ra  random_rk   r,   r*   rA   rA     "    Q<DDQG r,   z0https://github.com/pytorch/pytorch/issues/117971)rC   rD   rE   r   r^   rJ   	decoratorc                      t        j                  ddt         j                        j                  d      j	                  dd      S Nr6   i   r   r4   r  r8   rn  ra  r  r<   rk   r,   r*   rA   rA     1    Q5::>FFqIPPQRTWX r,   discontiguous)rC   rD   rE   r   r^   rN   rJ   r
  EmbeddingBagz$torch::nn::EmbeddingBagOptions(4, 3)c                  l    t        j                  ddt         j                        j                  d      S r  r  rk   r,   r*   rA   rA     r	  r,   mean)rC   rD   rE   r   r^   rN   rJ   c                      t        j                  ddt         j                        j                  d      j	                  dd      S r  r  rk   r,   r*   rA   rA     r  r,   )r4   rW   N       @FrR  ztorch::nn::EmbeddingBagOptions(4, 3)
                                .max_norm(std::nullopt).norm_type(2.).scale_grad_by_freq(false).mode(torch::kSum)c                  l    t        j                  ddt         j                        j                  d      S r  r  rk   r,   r*   rA   rA     r	  r,   rR  )r4   rW   Nr  Fmaxztorch::nn::EmbeddingBagOptions(4, 3)
                                .max_norm(std::nullopt).norm_type(2.).scale_grad_by_freq(false).mode(torch::kMax)c                  l    t        j                  ddt         j                        j                  d      S r  r  rk   r,   r*   rA   rA     r	  r,   r  EmbeddingBag_mean_padding_idxc                  2    t        j                  ddd      S )Nr4   rW   r6   padding_idxru   r  rk   r,   r*   rA   rA     s    BOOAqa@ r,   z3torch::nn::EmbeddingBagOptions(4, 3).padding_idx(1)c                  |    t        j                  t        j                  d      t        j                  d      g      S NrW   r8   stackra   rk   r,   r*   rA   rA     &    ennQ&79J%KL r,   )r   r   rE   r   r^   rJ   EmbeddingBag_sum_padding_idxc            	      :    t        j                  ddd dddd      S )Nr4   rW   r  FrR  r6   r  r  rk   r,   r*   rA   rA         BOOAq$E5VWX r,   ztorch::nn::EmbeddingBagOptions(4, 3)
                                .max_norm(std::nullopt).norm_type(2.).scale_grad_by_freq(false).mode(torch::kSum).padding_idx(1)c                  |    t        j                  t        j                  d      t        j                  d      g      S r  r  rk   r,   r*   rA   rA     r!  r,   EmbeddingBag_max_padding_idxc            	      :    t        j                  ddd dddd      S )Nr4   rW   r  Fr  r6   r  r  rk   r,   r*   rA   rA     r$  r,   ztorch::nn::EmbeddingBagOptions(4, 3)
                                .max_norm(std::nullopt).norm_type(2.).scale_grad_by_freq(false).mode(torch::kMax).padding_idx(1)c                  |    t        j                  t        j                  d      t        j                  d      g      S r  r  rk   r,   r*   rA   rA     r!  r,   EmbeddingBag_sparsec                  P    t        j                  dddt        j                        S )Nr4   rW   T)sparser   )ru   r  r8   rb   rk   r,   r*   rA   rA     s    BOOAqU\\R r,   zbtorch::nn::EmbeddingBagOptions(4, 3).sparse(true)._weight(torch::rand({4, 3}).to(torch::kFloat64))c                  L    t        j                  d      j                  dd      S NrP   r6   r8   ra   repeatrk   r,   r*   rA   rA          *11!Q7 r,   )r   r   rE   r   r^   has_sparse_gradientsc                  P    t        j                  ddt        j                  d      S )Nr4   rW   T)r   r+  )ru   r  r8   rb   rk   r,   r*   rA   rA     s    BLLAU\\$O r,   z_torch::nn::EmbeddingOptions(4, 3).sparse(true)._weight(torch::rand({4, 3}).to(torch::kFloat64))c                  L    t        j                  d      j                  dd      S r-  r.  rk   r,   r*   rA   rA     r0  r,   Embedding_sparse)r   rE   r   r   r^   r1  PixelShuffler  z!torch::nn::PixelShuffleOptions(3))r6   	   r4   r4   PixelUnshufflez#torch::nn::PixelUnshuffleOptions(3))r6   r6      r8  r8  nearest)rc   scale_factormodezF::InterpolateFuncOptions()
                            .size(std::vector<int64_t>({12})).scale_factor(std::nullopt).mode(torch::kNearest))r6   rP   r4   interpolate_nearest_1d)r   cpp_options_argsrF   r   r   rJ   )r   rP   r4   interpolate_nearest_1d_zero_dim)r   r=  rF   r   r   )r8  )r6   rP   rW   interpolate_nearest_tuple_1dg      @zF::InterpolateFuncOptions()
                            .size(std::nullopt).scale_factor(std::vector<double>({4.})).mode(torch::kNearest)interpolate_nearest_scale_1dlinear)rc   r:  r;  align_cornerszF::InterpolateFuncOptions()
                            .size(std::vector<int64_t>({12}))
                            .scale_factor(std::nullopt)
                            .mode(torch::kLinear)
                            .align_corners(false)interpolate_linear_1d)r4   zF::InterpolateFuncOptions()
                            .size(std::vector<int64_t>({4}))
                            .scale_factor(std::nullopt)
                            .mode(torch::kLinear)
                            .align_corners(false)interpolate_linear_tuple_1dzF::InterpolateFuncOptions()
                            .size(std::nullopt)
                            .scale_factor(std::vector<double>({4.}))
                            .mode(torch::kLinear)
                            .align_corners(false)interpolate_linear_scale_1dinterpolate_linear_1d_zero_dimzF::InterpolateFuncOptions()
                            .size(std::vector<int64_t>({12}))
                            .scale_factor(std::nullopt)
                            .mode(torch::kLinear)
                            .align_corners(true)#interpolate_linear_1d_align_cornerszF::InterpolateFuncOptions()
                            .size(std::nullopt)
                            .scale_factor(std::vector<double>({4.}))
                            .mode(torch::kLinear)
                            .align_corners(true))interpolate_linear_scale_1d_align_cornersrP   zF::InterpolateFuncOptions()
                            .size(std::vector<int64_t>({2, 2}))
                            .scale_factor(std::nullopt)
                            .mode(torch::kNearest))r6      r6   r6   %interpolate_nearest_2d_launch_configszF::InterpolateFuncOptions()
                            .size(std::vector<int64_t>({12, 12}))
                            .scale_factor(std::nullopt)
                            .mode(torch::kNearest))r6   rP   r4   r4   interpolate_nearest_2d)r8     zF::InterpolateFuncOptions()
                            .size(std::vector<int64_t>({12, 16}))
                            .scale_factor(std::nullopt)
                            .mode(torch::kNearest))r6   rP   rW   r4   interpolate_nearest_tuple_2dzF::InterpolateFuncOptions()
                            .size(std::nullopt)
                            .scale_factor(std::vector<double>({4., 4.}))
                            .mode(torch::kNearest)interpolate_nearest_scale_2d)r   rP   r4   r4   interpolate_nearest_2d_zero_dimbilinearzF::InterpolateFuncOptions()
                            .size(std::vector<int64_t>({12, 12}))
                            .scale_factor(std::nullopt)
                            .mode(torch::kBilinear)
                            .align_corners(false)interpolate_bilinear_2d interpolate_bilinear_2d_zero_dim)r4   r]   zF::InterpolateFuncOptions()
                            .size(std::vector<int64_t>({4, 6}))
                            .scale_factor(std::nullopt)
                            .mode(torch::kBilinear)
                            .align_corners(false))r6   rP   rP   rW   interpolate_bilinear_tuple_2dzF::InterpolateFuncOptions()
                            .size(std::nullopt)
                            .scale_factor(std::vector<double>({4., 4.}))
                            .mode(torch::kBilinear)
                            .align_corners(false)interpolate_bilinear_scale_2d)r  r  zF::InterpolateFuncOptions()
                            .size(std::nullopt)
                            .scale_factor(std::vector<double>({2., 2.}))
                            .mode(torch::kBilinear)
                            .align_corners(false)*interpolate_bilinear_scale_tuple_shared_2d)r        ?zF::InterpolateFuncOptions()
                            .size(std::nullopt)
                            .scale_factor(std::vector<double>({2., 1.}))
                            .mode(torch::kBilinear)
                            .align_corners(false)*interpolate_bilinear_scale_tuple_skewed_2dzF::InterpolateFuncOptions()
                            .size(std::vector<int64_t>({4, 6}))
                            .scale_factor(std::nullopt)
                            .mode(torch::kBilinear)
                            .align_corners(true)+interpolate_bilinear_tuple_2d_align_cornerszF::InterpolateFuncOptions()
                            .size(std::nullopt)
                            .scale_factor(std::vector<double>({2., 1.}))
                            .mode(torch::kBilinear)
                            .align_corners(true)8interpolate_bilinear_scale_tuple_skewed_2d_align_cornersbicubiczF::InterpolateFuncOptions()
                            .size(std::vector<int64_t>({12, 12}))
                            .scale_factor(std::nullopt)
                            .mode(torch::kBicubic)
                            .align_corners(false)interpolate_bicubic_2dinterpolate_bicubic_2d_zero_dimzF::InterpolateFuncOptions()
                            .size(std::vector<int64_t>({4, 6}))
                            .scale_factor(std::nullopt)
                            .mode(torch::kBicubic)
                            .align_corners(false)interpolate_bicubic_tuple_2dzF::InterpolateFuncOptions()
                            .size(std::nullopt)
                            .scale_factor(std::vector<double>({4., 4.}))
                            .mode(torch::kBicubic)
                            .align_corners(false)interpolate_bicubic_scale_2dzF::InterpolateFuncOptions()
                            .size(std::nullopt)
                            .scale_factor(std::vector<double>({2., 2.}))
                            .mode(torch::kBicubic)
                            .align_corners(false))interpolate_bicubic_scale_tuple_shared_2dzF::InterpolateFuncOptions()
                            .size(std::nullopt)
                            .scale_factor(std::vector<double>({2., 1.}))
                            .mode(torch::kBicubic)
                            .align_corners(false))interpolate_bicubic_scale_tuple_skewed_2dzF::InterpolateFuncOptions()
                            .size(std::vector<int64_t>({4, 6}))
                            .scale_factor(std::nullopt)
                            .mode(torch::kBicubic)
                            .align_corners(true)*interpolate_bicubic_tuple_2d_align_cornerszF::InterpolateFuncOptions()
                            .size(std::nullopt)
                            .scale_factor(std::vector<double>({2., 1.}))
                            .mode(torch::kBicubic)
                            .align_corners(true)7interpolate_bicubic_scale_tuple_skewed_2d_align_cornerszF::InterpolateFuncOptions()
                            .size(std::vector<int64_t>({12, 12, 12}))
                            .scale_factor(std::nullopt)
                            .mode(torch::kNearest))r6   rP   r4   r4   r4   interpolate_nearest_3d)r   rP   r4   r4   r4   interpolate_nearest_3d_zero_dim)r8  rL  rL  zF::InterpolateFuncOptions()
                            .size(std::vector<int64_t>({12, 16, 16}))
                            .scale_factor(std::nullopt)
                            .mode(torch::kNearest))r6   rP   rW   r4   r4   interpolate_nearest_tuple_3dzF::InterpolateFuncOptions()
                            .size(std::nullopt)
                            .scale_factor(std::vector<double>({4., 4., 4.}))
                            .mode(torch::kNearest)interpolate_nearest_scale_3d	trilineara   F::InterpolateFuncOptions()
                            .size(std::vector<int64_t>({12, 12, 12}))
                            .scale_factor(std::nullopt)
                            .mode(torch::kTrilinear)
                            .align_corners(false)interpolate_trilinear_3d!interpolate_trilinear_3d_zero_dim)r4   r]   r]   zF::InterpolateFuncOptions()
                            .size(std::vector<int64_t>({4, 6, 6}))
                            .scale_factor(std::nullopt)
                            .mode(torch::kTrilinear)
                            .align_corners(false))r6   rP   rP   rW   rW   interpolate_trilinear_tuple_3dg      @zF::InterpolateFuncOptions()
                            .size(std::nullopt)
                            .scale_factor(std::vector<double>({3., 3., 3.}))
                            .mode(torch::kTrilinear)
                            .align_corners(false)interpolate_trilinear_scale_3dr   )r   r=  rF   r   r   r   rJ   zF::InterpolateFuncOptions()
                            .size(std::vector<int64_t>({4, 6, 6}))
                            .scale_factor(std::nullopt)
                            .mode(torch::kTrilinear)
                            .align_corners(true),interpolate_trilinear_tuple_3d_align_cornerszF::InterpolateFuncOptions()
                            .size(std::nullopt)
                            .scale_factor(std::vector<double>({3., 3., 3.}))
                            .mode(torch::kTrilinear)
                            .align_corners(true),interpolate_trilinear_scale_3d_align_cornersr7   rO  zF::SoftmaxFuncOptions(-1))rP   rI  softmax_lastdimr6   )rP  r   z/F::SoftmaxFuncOptions(1).dtype(torch::kFloat64)softmax_lastdim_dtype)r   r=  rF   r   r   rQ   rJ   zF::SoftmaxFuncOptions(1))rP   rI  rP   rP   softmax_spatial_special)rP   rP   r4   r4   softmax_spatialsoftmax_spatial_dtypezF::SoftmaxFuncOptions(0)softmax_functional_dim0)r   r=  rF   r   rQ   r   rJ   rW   zF::SoftmaxFuncOptions(3)softmax_functional_dim3rk   softmax_functional_scalar)r   r=  rF   r   rQ   r   zF::LogSoftmaxFuncOptions(-1)log_softmax_lastdimzF::LogSoftmaxFuncOptions(1)log_softmax_spatial_speciallog_softmax_spatialzF::LogSoftmaxFuncOptions(0)log_softmax_dim0zF::LogSoftmaxFuncOptions(3)log_softmax_dim3log_softmax_scalarUnfoldc                  2    t        j                  dddd      S )Nr  r  r   r   ru   r|  rk   r,   r*   rA   rA   n	  s    BIIffffE r,   zPtorch::nn::UnfoldOptions({2, 2}).dilation({1, 1}).padding({0, 0}).stride({1, 1}))rP   r4   rW   rW   )r   r   rE   rF   r^   rQ   rJ   Foldc                  4    t        j                  ddddd      S Nr  r  r  r~  ru   r  rk   r,   r*   rA   rA   w	      BGGFFFFFK r,   zVtorch::nn::FoldOptions({3, 3}, {2, 2}).dilation({1, 1}).padding({0, 0}).stride({1, 1}))rP   rL  r4   Fold_no_batch_dim_inputc                  4    t        j                  ddddd      S r  r  rk   r,   r*   rA   rA   	  r  r,   )rL  r4   )r   r   rE   rF   r^   refrQ   rJ   Unfold_int_inputc                  2    t        j                  dddd      S )NrP   r6   r   r  rk   r,   r*   rA   rA   	  s    BIIaAq1 r,   z<torch::nn::UnfoldOptions(2).dilation(1).padding(0).stride(1)Fold_int_inputc                  4    t        j                  ddddd      S NrW   rP   r6   r   r  rk   r,   r*   rA   rA   	      BGGAq!Q2 r,   z=torch::nn::FoldOptions(3, 2).dilation(1).padding(0).stride(1)Fold_no_batch_dim_int_inputc                  4    t        j                  ddddd      S r  r  rk   r,   r*   rA   rA   	  r  r,   )r   r   rE   rF   r  r^   rQ   rJ   with_up_down_scalarPairwiseDistancec                  Z    t        j                  dd      t        j                  dd      fS Nr2   r3   r.  rk   r,   r*   rA   rA   	  !    %++b!,ekk"a.@A r,   )rC   r   rJ   c                  Z    t        j                  dd      t        j                  dd      fS )Nr2   r6   r3   r.  rk   r,   r*   rA   rA   	  r  r,   broadcast_lhs)rC   r   rN   rJ   c                  Z    t        j                  dd      t        j                  dd      fS )Nr2   r3   r6   r.  rk   r,   r*   rA   rA   	  s!    %++b!,ekk!Q.?@ r,   broadcast_rhs)g      ?r   TzDtorch::nn::PairwiseDistanceOptions().p(1.5).eps(1e-05).keepdim(true)c                  Z    t        j                  dd      t        j                  dd      fS r  r.  rk   r,   r*   rA   rA   	  r  r,   with_non_default_args)rC   rD   rE   r   rN   rJ   c                  V    t        j                  d      t        j                  d      fS )Nr3   r.  rk   r,   r*   rA   rA   	  s    %++a.%++a.9 r,   )rC   r   rG   rN   rJ   TransformerEncoderLayer)r4   rP   rL          ztorch::nn::TransformerEncoderLayerOptions(4, 2)
                                .dim_feedforward(16)
                                .dropout(0.0)r  relu_activationrR   )
rC   rD   rE   rF   rN   rH   rI   r_   r^   rJ   r4   r3   r  ztorch::nn::TransformerEncoderLayerOptions(4, 2)
                                .dim_feedforward(8)
                                .dropout(0.0)
                                .activation(torch::kGELU)gelu_activationg{Gz?)	rC   rD   rE   rF   r^   rN   rH   rI   rJ   TransformerDecoderLayer)r4   rP   r3   r  ztorch::nn::TransformerDecoderLayerOptions(4, 2)
                                .dim_feedforward(8)
                                .dropout(0.0)c                  ^    t        j                  ddd      t        j                  ddd      fS NrW   r4   rP   r   rk   r,   r*   rA   rA   	  %    %**Q1-uzz!Q/BC r,   )	rC   rD   rE   r   r^   rN   rH   rI   rJ   ztorch::nn::TransformerDecoderLayerOptions(4, 2)
                                .dim_feedforward(8)
                                .dropout(0.0)
                                .activation(torch::kGELU)c                  ^    t        j                  ddd      t        j                  ddd      fS r  r   rk   r,   r*   rA   rA    
  r  r,   Transformera  torch::nn::TransformerOptions()
                                .d_model(4)
                                .nhead(2)
                                .num_encoder_layers(2)
                                .num_decoder_layers(2)
                                .dim_feedforward(8)
                                .dropout(0.0)
                                .activation(torch::kReLU)c                      t        j                  ddd      t        j                  ddd      t        j                  dd      fS r  r   rk   r,   r*   rA   rA   
  s3    %**Q1-uzz!Q/BEJJqRSDTU r,   multilayer_codergQ?)rW   rS   ztorch::nn::LinearOptions(3, 5)c                  ,    t        j                  d      S r  r   rk   r,   r*   rA   rA   
  s    A r,   c                     t        j                  | j                  dd      |d   j                               j                  d      |d   z   S )Nr6   r7   r   )r8   r9   r;   r:   r=   s      r*   rA   rA   
  s<    UXXaffQmQqTVVX%F%K%KB%ORSTURV%V r,   )	rC   rD   rE   r   rG   rN   rH   rI   rJ   z5torch::nn::FlattenOptions().start_dim(-3).end_dim(-1))r7   )rW   r4   rS   )rC   rE   rD   rF   rG   rN   rJ   	Unflattenz'torch::nn::UnflattenOptions(-2, {2, 2})	LayerNorm)8   r  r  zMtorch::nn::LayerNormOptions({56, 56, 56}).eps(1e-5).elementwise_affine(false))r4   r  r  r  3d_no_affine_large_feature)	rC   rD   rE   rF   r  
check_evalgradcheck_fast_mode
check_halfrN   )reflectcircular	replicater  )ztorch::kReflectztorch::kCircularztorch::kReplicateztorch::kZerosr  {z, }r  r  c              #   &   K   | ]	  }|d z     ywr6   Nrk   .0r?   s     r*   	<genexpr>r  S
  s     $<qQU$<s   Convdztorch::nn::ConvzvdOptions(2, 3, 3)
                                        .stride(2)
                                        .padding(z)
                                        .dilation(1)
                                        .groups(1)
                                        .bias(true)
                                        .padding_mode()_stride2_pad2)
rC   rD   rE   rF   output_sizer  rN   rH   rI   rJ   )ELU
HardshrinkHardsigmoidHardtanh	Hardswish	LeakyReLU
LogSigmoidPReLUReLUReLU6rO   SELUCELUGELUGLUr   SiLUMishSoftplus
SoftshrinkSoftsignTanh
Tanhshrink	Thresholdr  )r  )rD   rJ   r  rD   r  )r^   	check_jitrJ   r  )rQ   rJ   r  rJ   r  r  r  r  r  r  r  r  r  r  r  )r  r   r  r  r  r  r  !non_linear_activations_extra_info)rC   rF   rG   rN   test_cpp_api_parityc                 6   |rt        j                  |      || z
  z  }n||j                         | z
  z  }|dk(  r|j                         S |dk(  r|j	                         S |dk(  r5|j                         dk7  r"|j	                         |j                  d      z  S |S )Nr  rR  	batchmeanr   )r8   r   r   r  rR  rP  rc   )r  rJ  r!   r  r)   s        r*   kldivloss_referencer  
  s    6"fun56::<%/0F{{}	e	zz|	k	!fjjla&7zz|fkk!n,,Mr,   c                    | j                         dk\  sJ | j                  d      }| j                  d      }|f| j                         dd  z   }t        j                  |      j	                  |       }|$t        j
                  |      j	                  |       }d}	t        |D 
cg c]  }
t        |
       c}
 D ]X  }||   }||k(  rdn||   j                         }t        |      }|j                  d|       | t        |          |z  ||<   |	|z  }	Z |dk(  r|j                         |	z  S |dk(  r|j                         S |S c c}
w )NrW   r   r6   rP   r  r  rR  )rP  rc   r8   r  r   onesr   rangeitemr  insertr  rR  )r  rJ  r.   ru  r!   NCout_sizeoutputtotal_weightrc   tupt_nxnorminput_indexs                  r*   nlllossNd_referencer  
  sB   99;!

1A

1Atejjl12&&H[["**51F~A&&u-L:t:; c{!T)rvd|/@/@/B3i1d#U;/0047s Fzz|l**	e	zz|M ;s   $Ec                    | j                         dk\  sJ t        j                  | d      } | j                  d      }|$t        j                  |      j                  |       } |j                  d|gd | j                  dd  D         }|dkD  r|dk  sJ |d|z
  z  ||z  z   }| |z  |z  j                  d       }|dk(  r|j                         S |dk(  r|j                         S |S )	NrP   r6   c              3       K   | ]  }d   ywr  rk   r  r@   s     r*   r  z;cross_entropy_loss_prob_target_reference.<locals>.<genexpr>
  s      <q <   r  rV  rO  r  rR  )
rP  r8   log_softmaxrc   r  r   r;   shaperR  r  )r  rJ  r.   r!   label_smoothingr  r  s          r*   (cross_entropy_loss_prob_target_referencer  
  s    99;!eQ'E

1A~A&&u-V[[A= <EKKO <=F#%%%A/0?Q3FFv~&+++22FF{{}	e	zz|Mr,   c                 (   t        j                  | d      }t        j                  |||||      }|dk(  r|S d|cxk  rdk  sJ  J t        j                  | d      } | j	                  d      }|+|  |j
                  d|gd | j                  dd  D         z  } t        j                  | d       }	||k(  }
|	j                  |
d       |dk(  r|bt        j                  |	      |j                  d|j                  |
j                               j                               j                         z  }nPt        j                  |	j                  |
j                                     }n|d	k(  rt        j                  |	      }n|	}d|z
  |z  |||z  z  z   S )
Nr6   rt  r  rV  c              3       K   | ]  }d   ywr  rk   r  s     r*   r  z>cross_entropy_loss_indices_target_reference.<locals>.<genexpr>
  s     +G!A+Gr  rP   r  r   rR  )r8   r  r~   r`  rc   r;   r  rR  masked_fill_gathermasked_selectlogical_notrZ   r  )r  rJ  r.   ru  r!   r  log_softmax_inputnlllossr  smooth_lossignore_maskrets               r*   +cross_entropy_loss_indices_target_referencer   
  s   ))%3jj!G #'C'''''eQ'E

1AAqH+Gu{{12+GHH99UA&&KL(K[#.F ))K(6==F<P<PQ\QhQhQj<k<s<s<u+v+z+z+||C**[66{7N7N7PQRC	e	ii$7*SOa4G-HHHr,   c                 v    | j                   |j                   k(  rt        | ||||      S t        | |||||      S )N)r.   r!   r  )r.   r!   ru  r  )r  r  r   )r  rJ  r.   ru  r!   r  s         r*   cross_entropy_loss_referencer  
  sK    {{fll"7+- 	- ;6&I%
 	
r,   c           
          d }t        | |      D cg c]  \  }} |||||       }}}t        | \  }	}
| j                  |	      }|dk(  rt        |      t        |
      z  S |dk(  rt        |      S |S c c}}w )Nc                 :    ||k(  ry|dn||   }| |    |z  }||fS )Nr~  r6   rk   )r  rJ  r.   ru  r  r)   s         r*   nll_loss_helperz*nllloss_reference.<locals>.nll_loss_helper  s6    \!Nqv-$&~r,   r  rR  )zip
new_tensorrR  )r  rJ  r.   ru  r!   r  r>   r:   losses_and_weightslossesr/   losses_tensors               r*   nllloss_referencer    s     '*%&8:"a *!QE : :-.OFG$$V,MF=!CL00	e	=!!:s   A5c                    | |z
  j                         }||k\  j                  |      }||k  j                  |      }|dk(  r|}n||d|z  z
  z  |dz  |dz  z  |z  z   }|dk(  r|j                         S |dk(  r|j                         S |S )Nr   r  rP   r  rR  )r2  r   r  rR  )r  rJ  r!   r  abs_diffge_beta_masklt_beta_maskr  s           r*   smoothl1loss_referencer  %  s    ##%H$--h7LtO,,X6LqyC$J!67,:LPX\]P]:^ae:eeF{{}	e	zz|Mr,   c                     | |z
  j                         }||k\  }||k  }||z  |d|z  z
  z  |dz  |dz  z  z   }|dk(  r|j                         S |dk(  r|j                         S |S )Nr  rP   r  rR  )r2  r  rR  )r  rJ  r!   r  r  ge_delta_masklt_delta_maskr  s           r*   huberloss_referencer  5  s    ##%H&M%MU"hu&<=PS@SW_cdWd@eeFF{{}	e	zz|Mr,   c           	          g }|D ]  }|dk  r n|j                  |        d}|D ]<  }t        dt        |             D ]"  }||vs|t        dd| |   z
  | |   z         z  }$ > |S )Nr   r6   )appendr  lenr  )r  rJ  targetstarget_indexrR  r>   s         r*   _multilabelmarginloss_referencer  A  s    G %!|$%
 C Bq#e*% 	BAs1a%"55a@AA	BB
 Jr,   c                    | j                         }| j                         dk  r|j                         dk  sJ | j                         dk(  r| j                  d      n| j                  d      j                  d      } |j                         dk(  r|j                  d      n|j                  d      j                  d      }| j                  d      }| j                  d      }| j                  |      j	                         }t        d|      D ]  }t        | |   ||         ||<    |dk(  r|j                         |z  S |dk(  r|j                         |z  S |dk  r|j                         |z  S ||z  S NrP   r6   r   r  rR  )
rP  r  rc   newzero_r  r  r  rR  r  )r  rJ  r!   	input_dimnrP  r  r>   s           r*   multilabelmarginloss_referencer!  Q  sO   		Iyy{Qzz|a&+iikQ&6"EOOA<N<X<XYZ<[(.

(9!!!$v?O?OPQ?R?\?\]^?_

1A
**Q-CYYq\!F1a[ I3E!HfQiHq	I F{{}s""	e	zz|c!!	Q ~~#%%|r,   c                     || z
  j                  d      j                  |       }t        j                  |dk(  | |      }|dk(  r|j	                         S |dk(  r|j                         S |S )Nr   r  r6   r  rR  )r!  r   r8   wherer  rR  )r  rJ  r;  r!   margin_clampr  s         r*   hingeembeddingloss_referencer%  k  sg    UN))a)088?L[[1e\:FF{{}	e	zz|Mr,   c                     d|  |z  j                         z   j                         }|dk(  r|j                         S |dk(  r|j                         S |S )Nr6   r  rR  )r   r   r  rR  )r  rJ  r!   r  s       r*   softmarginloss_referencer'  v  sQ    E6F?''))..0FF{{}	e	zz|Mr,   c           	          |)| j                  t        |             j                  d      }d}t        dt        |             D ],  }||k7  s	|||   t	        d|| |   z
  | |   z         |z  z  z  }. |S )Nr6   r   )r  r  fill_r  r  )r  
target_idxr?   r;  r.   r  r>   s          r*   _multimarginloss_referencer+    s    ~3u:&,,Q/F1c%j! b
?fZ(CFU:=N4NQVWXQY4Y,[_`,`aaFb Mr,   c                 l   | j                         dk  rD| j                         dk(  r| j                  d      n| j                  d      j                  d      } |j                         }|j                         dk(  r|j                  d      }| j                  d      }| j                  d      }| j                  |      }	t	        d|      D ]  }
t        | |
   ||
   |||      |	|
<    |dk(  r|	j                         |z  S |dk(  r|	j                         |z  S |dk(  r|	j                  d      |z  S |	|z  S r  )	rP  r  rc   r  r  r+  r  rR  r  )r  rJ  r?   r;  r.   r!   
target_dimr   rP  r  xs              r*   multimarginloss_referencer/    s   yy{Q&+iikQ&6"EOOA<N<X<XYZ<[Jzz|q!!!$

1A
**Q-CYYq\F1a[ W.uQxAvvVq	W F{{}s""	e	zz|c!!	q~~a 3&&C<r,   c                     d }t        j                  |dk(  d || |      z
   || |      |z
  j                  d            }|dk(  r|j                         S |dk(  r|j	                         S |S )Nc                 8   | j                  | j                  d            }t        d| j                  d            D ][  }| |   ||   z  j                         | |   | |   z  j                         dz   ||   ||   z  j                         dz   z  dz  z  ||<   ] |S )Nr   g-q=r  )r  rc   r  rR  )abcosr>   s       r*   _cosz+cosineembeddingloss_reference.<locals>._cos  s    eeAFF1Iq!&&)$ 	tAdQqTk&&(qtad{.?.?.AE.IqQRtVWXYVZ{N_N_NadiNi-jor,rsCF	t
r,   r6   r   r  r  rR  )r8   r#  r!  r  rR  )input1input2rJ  r;  r!   r5  r  s          r*   cosineembeddingloss_referencer8    st     [[1a$vv*>&>ffAUX^A^@e@ejk@e@lmFF{{}	e	zz|Mr,   c                 T   t        j                  | |||      }t        j                  | |||      }	|r.t        j                  ||||      }
t        j                  |	|
      }	t        j                  ||z   |	z
  d      }|dk(  r|j	                         S |dk(  r|j                         S |S )Nr  r  r  rR  )r8   pairwise_distancer  r!  r  rR  )anchorpositivenegativer;  r?   epsswapr!   d_pd_nd_sr  s               r*   tripletmarginloss_referencerC    s    

!
!&(As
;C

!
!&(As
;C%%h!SAiiS![[#+5FF{{}	e	zz|Mr,   c                     | | |z
  z  |z   j                  d      }|dk(  r|j                         S |dk(  r|j                         S |S )Nr   r  r  rR  )r!  r  rR  )r6  r7  rJ  r;  r!   r  s         r*   marginrankingloss_referencerE    sR    g&)F299a9@FF{{}	e	zz|Mr,   c                 @   t        j                  |t         j                        }t        j                  |t         j                        }| j                  }| j	                         } |j                         }|j                  d      }g }t        | j                  d            D ]|  }	||	   j                         }
||	   j                         }||	   j                         }|j                  d|z  dz   f|      }|j                         dk(  r||	d |f   |dd d<   n|||z
  | |dd d<   | d |
|	f   j                         }| j                  |dz  dz   f      }|d|f   |d<   |d|d   f   |d<   |d d |dd  k7  }t        d|
      D ]b  }|j                         }|dd xxx |d d z  ccc |dd xxx t        j                  ||d d |j                  d            z  ccc |||f   |z  }d |j                  |dd  j!                         j#                         d             t        j$                  |d      }|dk(  r9||j'                  |j                  |j(                        z  j+                         }n|d	k(  r|j!                         }|j'                  |      }|S )
Nr   r   r6   rP   r  r7   r  r   devicerR  )r8   	as_tensorra  r   rb   cumsumr  rc   r  new_fullrP  r   	new_zeroscloner#  r  rR  r   catr   rH  r  )	log_probsr  input_lengthstarget_lengthsblankr!   dtcum_target_lengthsr	  r>   input_lengthtarget_lengthcum_target_lengthtargets_primeprobsalpha
mask_thirdr:   
alpha_nextr  s                       r*   ctcloss_referencer]    s   OOMDM__^5::FN	B  "IllnG'..q1F9>>!$% 5$Q',,.&q)..0.q1668((!m*;a*?)A5I;;=A")!^m^*;"<M!$Q$")*;m*KL]"^M!$Q$-<-*+//1##]Q%6%:$<=E?aM!,,-a#CR(M!",==
q,' 	9AJqrNeCRj(NqrNekk*eCRj%//RSBTUUN!]*+j8E		9
 	uRSz~~'++-d334'5( YYvq!FF>,,6<<,VV\\^	e	YYr]FMr,   )r  )r   r  ri  r  r  r  r
  r0  rE  re  CosineEmbeddingLossTripletMarginLossMarginRankingLossCTCLossCrossEntropyLossr{  r   c                      | d   }d }fd | dd D cg c]
  } ||       c}      } || }t        |      }|dk(  r|j                  d      S |S c c}w )zReference function for criterion supporting no batch dimensions.

    The criterion is passed the input and target in batched form with a single item.
    The output is squeezed to compare with the no-batch input.
    r7   c                     t        | t        t        f      r| D cg c]  }|j                  d       c}S | j                  d      S c c}w rM   r  r  s     r*   r  z:single_batch_reference_criterion_fn.<locals>.unsqueeze_inp  r  r  c                     g }t        | t        t        f      r | D ]  }|j                   |              |S |j	                  |        |S rj   )r  r  r  extendr  )xsr)   r.  rZ   s      r*   rZ   z4single_batch_reference_criterion_fn.<locals>.flatten  sO    b4-( *gaj)*  MM"r,   Nr|   r   )r+   r  )rm   	criterionr  r  single_batch_input_argsr  r!   rZ   s          @r*   #single_batch_reference_criterion_fnrj    sr     RI 
 &cr&S}U';&ST/0Fi(IF~~a  M 'Ts   A)L1LossMSELossPoissonNLLLossr  r  )r|   r  rR  _no_batch_dim_)namec                 8     t        t        |       t              S Nr}   r$   ru   r!   ro  rm   s     r*   rA   rA   0      ->WR->-S r,   )r   r   rF   target_sizerG   r  rJ   KLDivLoss_no_batch_dim_c                  6    t        j                  t              S rq  )ru   r   r!   rk   r,   r*   rA   rA   =  s    BLL9= r,   c                  H    t        j                  d      j                         S Nr  r   rk   r,   r*   rA   rA   >  s    D)--/ r,   c                  ,    t        j                  d      S ry  r   rk   r,   r*   rA   rA   ?  s    %**T* r,   )r   r   r   	target_fnrG   r  rJ   BCELossc                  r    t        j                  t        j                  dt         j                              S Nr6  r   )r8   r   r   rb   rk   r,   r*   rA   rA   L  s    ekk!5<<@A r,   c                      t        j                  dt         j                        j                  d      j	                  t         j                        S )Nr6  r   r   )r8   r   rb   r   r   rk   r,   r*   rA   rA   M  s/    AU\\255a8;;ELLI r,   BCEWithLogitsLossc                  L    t        j                  dt         j                        S r~  r8   r   rb   rk   r,   r*   rA   rA   O  s    %++au||"D r,   c                  L    t        j                  dt         j                        S r~  r  rk   r,   r*   rA   rA   O  s    ekkZ[chcocoNp r,   r0  c                  L    t        j                  dt         j                        S r~  r  rk   r,   r*   rA   rA   P  s    5;;q#E r,   c                  6    t        j                  g ddz        S N)r7   r6   r6   rW   r8   tensorrk   r,   r*   rA   rA   P  s    u||\fij\jOk r,   r
  c                  L    t        j                  dt         j                        S )Nr4   r   r  rk   r,   r*   rA   rA   Q  s    U[[%,,%G r,   c                  0    t        j                  g d      S )N)rW   r   r7   r6   r  rk   r,   r*   rA   rA   Q  s    QVQ]Q]^kQl r,   rE  c                  L    t        j                  dt         j                        S r~  r  rk   r,   r*   rA   rA   R  s    u{{1ELLA r,   c                  6    t        j                  g ddz        S r  r  rk   r,   r*   rA   rA   R  s    5<<XbefXfKg r,   ri  c                  v    t        j                  t        j                  dt        j                        d      S )NrW   r   r   rO  )r~   r  r8   r   rb   rk   r,   r*   rA   rA   S  s     ekk!5<<&HaP r,   c                  ,    t        j                  d      S rY   r  rk   r,   r*   rA   rA   S  s    Z_ZfZfghZi r,   r^  c                      t        j                  dt         j                        t        j                  dt         j                        fS r~  r  rk   r,   r*   rA   rA   V  )    Qell3U[[%,,5WX r,   c                  L    t        j                  dt         j                        S )Nr6   r   )r8   r  rb   rk   r,   r*   rA   rA   W  s    Qell3 r,   r`  c                  V    t        j                  d      t        j                  d      fS r  r.  rk   r,   r*   rA   rA   Z  s    5;;r?EKKO"D r,   c                  H    t        j                  d      j                         S r  )r8   r   signrk   r,   r*   rA   rA   Z  s    ekkZ\oNbNbNd r,   r_  c                      t        j                  dt         j                        t        j                  dt         j                        fS r~  r  rk   r,   r*   rA   rA   ^  r  r,   c                  L    t        j                  dt         j                        S r~  r  rk   r,   r*   rA   rA   _  s    AU\\2 r,   MultiLabelSoftMarginLossc                  L    t        j                  dt         j                        S r~  r  rk   r,   r*   rA   rA   a  s    Qell)K r,   c                  ,    t        j                  d      S )Nr6  r.  rk   r,   r*   rA   rA   a  s    UZU`U`abUc r,   r^   ,classification_criterion_no_batch_extra_info)r|  r  r0  ri  rE  c                 8     t        t        |       t              S rq  rr  rs  s     r*   rA   rA   s  rt  r,   c                      |        S rj   rk   fs    r*   rA   rA   t  s    AC r,   c                      |        S rj   rk   r  s    r*   rA   rA   u  s    ac r,   )r   r   r   r{  rG   r  
has_parityc                   |   e Zd Zed        Zedej                  deeej                     eej                     f   fd       Z
edej                  ddfd       Ze	 ddej                  dedej                  d	eej                  eej                     f   d
ef
d       Zd Zd Zd ZddefdZddefdZddefdZy)
NNTestCasec                     t         rj   NotImplementedError)rl   rm   ro   s      r*   _forwardzNNTestCase._forward      !!r,   r  returnc                     t         rj   r  rl   r  s     r*   _get_parameterszNNTestCase._get_parameters  r  r,   Nc                     t         rj   r  r  s     r*   _zero_grad_parametersz NNTestCase._zero_grad_parameters  r  r,   r  r  grad_outputcreate_graphc                     t         rj   r  )rl   r  r  r  r  r  s         r*   	_backwardzNNTestCase._backward  s
    
 "!r,   c                     t        |t              rt         fd|D              S t        |t              r |D cg c]  } j                  |       c}S t	        j
                  |j                               S c c}w )Nc              3   B   K   | ]  }j                  |        y wrj   )	_jacobian)r  elemnum_outrl   s     r*   r  z'NNTestCase._jacobian.<locals>.<genexpr>  s     I4g6Is   )r  r  r  r  r8   r  nelement)rl   r  r  r  s   ` ` r*   r  zNNTestCase._jacobian  sb    eU#I5IIIt$>CDdDNN41DD;;u~~/99 Es   A<c                      t        |t        j                        r<|j                  r|j	                         j                  d      S |j                  d      S t         fd|D              S )Nr7   c              3   @   K   | ]  }j                  |        y wrj   )_flatten_tensors)r  r2  rl   s     r*   r  z.NNTestCase._flatten_tensors.<locals>.<genexpr>  s     =a..q1=   )r  r8   r  	is_sparseto_denser;   r  )rl   r.  s   ` r*   r  zNNTestCase._flatten_tensors  sN    a&{{zz|((,,vvbz!=1===r,   c                    t        |t        j                        rO|j                  rB|j                  5|j                  j                          |j                  j                          y y y |D ]  }| j                  |        y rj   )r  r8   r  r   gradr  detach__zero_grad_input)rl   r  r>   s      r*   r  zNNTestCase._zero_grad_input  si    eU\\*""uzz'=

  "

""$ (>"  )%%a()r,   c                    | j                  ||      }|j                         }|r&| j                  ||      }t        t	        |            }|r:t        d | j                  |      d   D              }	t        j                  |	|      }
t        |      D ]%  }| j                  |      \  }}t        ||      D cg c]  \  }}|t        j                  |      n|  }}}t        j                  |      }|j                  d      }d||<   |r| j                  |       |r| j                  |       | j                  ||||      }|rCt        t	        |            D ]+  \  }}|j!                         j                  d      |d d |f<   - |st        j"                  | j%                  |      d      
d d |f<   ( d}|r|fz  }|r|
fz  }|S c c}}w )Nc              3   <   K   | ]  }|j                           y wrj   )numelr  s     r*   r  z2NNTestCase._analytical_jacobian.<locals>.<genexpr>  s     O!AGGIO   r   r7   r6   rk   )r  r  r  r  r   rR  r  r8   r  r  r  
zeros_liker;   r  r  r  
contiguousrN  r  )rl   r  r  jacobian_inputjacobian_parametersr  r  jacobian_inpflat_jacobian_input	num_paramjacobian_paramr>   paramd_paramr?   r  d_out
flat_d_outd_input
jacobian_xd_xress                         r*   _analytical_jacobianzNNTestCase._analytical_jacobian  s   vu-oo'>>%=L"&}\'B"COt/C/CF/KA/NOOI"[[K@N{# 	TA!11&9NE7MPQVX_M`a6Aqaiu''*Q>aGa$$V,EBJJqM"**62%%e,nnVUFEBG'*+>g@V'W AOJ'*~~'7'<'<R'@Jq!t$A"',yy1F1Fw1OQR'Sq!t$)	T, )+<= C>?"C
3 bs   9#Gc                       fd}d}|r|t        ||d      fz  }|r] j                        \  }}g }	|D ](  }
t        |||
d      }|	j                  |d   d          * |t        j                  |	d      fz  }|S )Nc                  D    j                  |       j                         S rj   )r  detach)r  r  rl   s    r*   fwz*NNTestCase._numerical_jacobian.<locals>.fw  s    ==/6688r,   rk   ư>)r>  )rJ  r>  r   )r   r  r  r8   rN  )rl   r  r  r  r  r  r  r  r@   to_catr?   jacobians   ``          r*   _numerical_jacobianzNNTestCase._numerical_jacobian  s    	9 )+*2u$?@@C++F3HE1F .22uQDQhqk!n-. EIIfa(**C
r,   c                 
   t        | j                  |      d         }| j                  ||||      }| j                  ||||      }t	        t        |            }t	        t        |            }g }	t        ||      D ]W  \  }
}|
j                         dk7  s|	j                  |
j                  |d      j                         j                                Y t        |	      dkD  r | j                  t        |	      t               y y )Nr   r7   )rZ  )boolr  r  r  r  r   r  r  r  r  r2  r  r  assertLessEqual	PRECISION)rl   r  r  r  r  
analytical	numericalanalytical_tnumerical_tdifferencesr2  r   s               r*   check_jacobianzNNTestCase.check_jacobian  s    "4#7#7#?#BC..vunNab
,,VUNL_`	M*56=34k2 	CDAqwwyA~""155"5#5#9#9#;#?#?#AB	C {a  [!19=  r,   F)TTT)rq   rr   rs   r   r  ru   rv   r   r   	Parameterr  r  r   r8   r  r   r   r  r  r  r  r  r  r  r  rk   r,   r*   r  r    s    " " "bii "E$r||:LdSUS_S_N`:`4a " " "BII "$ " "  (-"		 ")"38<<"$U\\8ELL3I%IJ" !%" ":>)(2B (T1A ">,< >r,   r  c                   \    e Zd Zh dZddZd Zd Zed        Zed        Z	d Z
dd	Zd
 Zy)TestBase>   r  
extra_argsrD   Nc                     || _         || _        || _        || _        | j                  D ]?  }||vs|dz   |vs|dz   |vs|dv rd||<   "t        | j                          d| d       || _        i | _        y )N_fn_size>   r  rD   rk   z
: Specify z5 by a value, a function to generate it, or it's size!)	rN   r   r   rG   _required_arg_names
ValueErrorget_name_extra_kwargs
_arg_cache)rl   r   rN   rG   r   ro   ro  s          r*   __init__zTestBase.__init__  s    	 &(,, 	@D6!dUl&&@TG^[aEa==#%F4L$'8
4&H}%~	@ $r,   c                     | j                   d| j                   z   S d| j                  j                  z   }| j                  r|d| j                  z   z  }|S )Ntest_r@   )r   r   rq   rN   )rl   	test_names     r*   r  zTestBase.get_name  sP    ==$T]]**d..777	99tyy(Ir,   c                      t        |t        j                        r|S t        |      r t	        |       fd|D              S |S )Nc              3   @   K   | ]  }j                  |        y wrj   )_unpack)r  vrl   s     r*   r  z#TestBase._unpack.<locals>.<genexpr>  s     >1t||A>r  )r  r8   r  r   type)rl   values   ` r*   r  zTestBase._unpack  s:    eU\\*L4;>>>>Lr,   c                 &    | j                  dd      S )NrD   T_get_argrl   s    r*   rD   zTestBase.constructor_args  s    }}/66r,   c                 &    | j                  dd      S )Nr  Tr  r  s    r*   r  zTestBase.extra_args!  s    }}\400r,   c           
         || j                   v sJ || j                  vr|dz   }|dz   }|| j                  v r| j                  |   | j                  |<   n|| j                  v r" | j                  |          | j                  |<   nV|| j                  v s!J d| d| d| d| j                                 fd | j                  |         | j                  |<   |r| j	                  | j                  |         S | j                  |   S )Nr  r  z	Missing `z`, `z` or `z` for c                     t        | t              r| D cg c]
  } |       c}S t        | t        j                        r| j	                         S t        j
                  |       S c c}w rj   )r  r  r8   r  rb   r   )sizessmap_tensor_sizess     r*   r
  z+TestBase._get_arg.<locals>.map_tensor_sizes4  sS    !%.=BC 0 3CC#E5<<8$||~-${{511	  Ds   A')r  r  r  r  r  )rl   ro  unpackfn_name	size_namer
  s        @r*   r  zTestBase._get_arg%  s   t/////t&UlGwIt)))(,(:(:4(@%D...(C(:(:7(C(E% D$6$66 \vT)F7)6$--/IZ[\62 )99K9KI9V(W%6<t||DOOD12W$//RVBWWr,   c                 &    | j                  d|      S )Nr  r  )rl   r  s     r*   
_get_inputzTestBase._get_input@  s    }}Wf--r,   c                     t         rj   r  )rl   	test_cases     r*   __call__zTestBase.__call__C  s    !!r,   ) NNr  )rq   rr   rs   r  r  r  r  propertyrD   r  r  r  r  rk   r,   r*   r  r    sO    E 7 7 1 1X6."r,   r  c            	       l     e Zd Zededej                  dedefd       Z fdZd Z	d Z
d	 Zd
 Z xZS )
ModuleTestr  r  r  r  c                     t         rj   r  )rl   r  r  r  s       r*   _do_testzModuleTest._do_testI  r  r,   c                    t        |   |i | |j                  dd      | _        |j                  dd      | _        |j                  dd      | _        |j                  dd      | _        |j                  dd      | _        |j                  dd	      | _        |j                  d
d      | _	        |j                  dd       | _
        | j                  t        j                         | _
        y y )Nr  TrQ   r   r^   !FIXME_no_cuda_gradgrad_comparisonFr   g-C6*?check_forward_onlyrJ   )superr  getr  should_test_cudashould_test_pickler^   r  r   r  rJ   r8   get_default_dtyperl   rm   ro   	__class__s      r*   r  zModuleTest.__init__M  s    $)&)$jj)94@ &

; ="(**Xt"<$jj)94@JJ:EB 	.K6"(**-A5"I#ZZ>%!&!8!8!:D &r,   c           	      j   t        | j                        5   | j                  | j                   }| j	                         }| j
                  a|j                  ||      }t        |      }t        |      }| j                  ||j                  |      d   |      }|j                  ||d       | j                  r
	 d d d        y | j                  |||       | j                  rt        j                         5 }|j                  ||       t        j                   ||       |j#                  d       t        j$                  |d      }	|j                  |j                  ||      |j                  |	|             d d d        | j'                  |||       d d d        y # 1 sw Y   %xY w# 1 sw Y   y xY w)Nr   F)exact_dtype)weights_only)r   rJ   r   rD   r  rG   r  r   r  assertEqualr  test_noncontigr  tempfileTemporaryFiler8   saveseekloadr  )
rl   r  r  r  out	ref_input
ref_moduleexpected_outr  module_copys
             r*   r  zModuleTest.__call__[  s   t112 	4%T%%t'<'<=FOO%E  ,((7$UO	%f-
#00I<U<UV\<]^_<`blm%%c<U%K&&	4 	4 	659&&++- u&&vu5JJvq)FF1I"'**QU"CK)))*<*<VU*KYM_M_`kmrMstu MM)VU31	4 	4 u u!	4 	4s+   B$F)3F)6BF9F)F&	"F))F2c                 Z    t        |t              r|D cg c]  } j                  |       c}S t        |t              rt         fd|D              S |}|j	                         }|}t        |      D ]  }|j                  |      dkD  s|dz   } n t        j                  t        j                  |      |g|      j                  |d      j                         }|j                         dk(  s%|j                         dk(  s|j                         rJ |j                  |_        |S c c}w )Nc              3   @   K   | ]  }j                  |        y wrj   )noncontiguize)r  orl   s     r*   r  z+ModuleTest.noncontiguize.<locals>.<genexpr>z  s     <1++A.<r  r6   r   )r  r  r4  r  rP  r  rc   r8   r   
empty_likeselectr  r  is_contiguousr   )rl   objr5  r  ndimrP  r  	noncontigs   `       r*   r4  zModuleTest.noncontiguizev  s   c4 367aD&&q)77U#<<<<zz|
 t 	A{{1~!!e	 KK!1!1&!96 BCHOOPSUVW^^`	 A%):a)?yG^G^G```"("6"6	# 8s   D(c           	      X   t        |t        j                        r|j                         dk(  ry t	        d |D              ry |j                  |       |j                  |       t               5  |j                  ||      }t        |dd      r|d   }|j                  |j                        j                         }|j                         }t        |j                  ||||            }t        |j!                  |      d         }d d d        | j#                  |      }| j#                        }	t%        dd      D ]  \  }
}|
r|n|}t        |r|n|	      }|j                  |       |j                  |       t               5  |j                  ||      }t        |dd      r|d   }|j                  ||||      }|j'                  |       |j'                  |d	d
       |j'                  |j!                  |      d          d d d         y # 1 sw Y   xY w# 1 sw Y   xY w)Nr   c              3   x   K   | ]2  }t        |t        j                        s|j                         d k(   4 yw)r   N)r  r8   r  rP  )r  r>   s     r*   r  z,ModuleTest.test_noncontig.<locals>.<genexpr>  s'     JjELL.Iquuw!|Js   ::return_indicesFr6   TFrP   )r/  g-C6?atolrtol)r  r8   r  rP  anyr  r  r   r  r$   r  r  normal_rM  r   r  r  r4  r   r&  )rl   r  r  r  r  r  r  r  nc_inputnc_grad_outputcontig_icontig_gr>   gor-  r  s                   r*   r'  zModuleTest.test_noncontig  s   eU\\*uyy{a/?JUJJ''/""5) 	E''6Fv/7 **V\\2::<K\\^Fy2265&+VWGy88@CDG	E %%e,++K8")-"B 	UHh!xA +~FB++F3&&q)!# U((36#3U;a&C **61c2>%%c62%%dG$Q%G%%i&?&?&G&JGTU U	U	E 	E$U Us   .BHBH H H)	c           	      
   t         r| j                  st        j                  d      t	        | j
                        5  | j                         }t        j                  t        j                  i}t        |t              r|n|f}t        d |D              }t        ||      } | j                  | j                   } | j                  | j                   j                         j!                         }|j#                  |      }	|j#                  |      }
t%        |	d   |
d         D ]   \  }}|j&                  j)                  |       " |j+                  |       |j+                  |       |j-                  |       |j-                  |       |j/                  ||      }|j/                  ||      }t1        |dd      r
|d   }|d   }|j3                  ||| j4                  dd       t7        d      D ]  }|j9                         j;                         }|j=                  |      }|j?                  ||||      }|j?                  ||||      }|j3                  ||| j4                  dd       t%        |	d	   |
d	         D ]$  \  }}|j3                  ||| j4                  d
       &  | j@                  r| jB                  s || } || }t1        |dd      r
|d   }|d   }t        jD                  |d      }|j=                  |      jG                         }d|_$        t        jJ                  jM                  ||t        |jO                               z   |d      }t        jJ                  jM                  ||t        |jO                               z   |d      }t%        ||      D ]%  \  }}|j3                  ||| j4                  dd       ' |rc|jQ                         jS                         tQ        d |D              z   }|jQ                         jS                         tQ        d |D              z   }nF|jQ                         tQ        d |D              z   }|jQ                         tQ        d |D              z   }t        jJ                  jM                  |||fz   t        |jO                               z   d      }t        jJ                  jM                  |||fz   t        |jO                               z   d      }|j3                  | j4                  dd       t%        ||      D ]%  \  }}|j3                  ||| j4                  dd       ' | jU                  |||       d d d        y # 1 sw Y   y xY w)NExcluded from CUDA testsc              3      K   | ]6  }t        |t        j                        xr |j                  j                   8 y wrj   )r  r8   r  r   
is_complexr  r:   s     r*   r  z'ModuleTest.test_cuda.<locals>.<genexpr>  s.     &s^_z!U\\'B'YqwwGYGY'Y&ss   <>)type_mapr   r>  FrA  rB  r$  rS   r6   r@  T)r   )r  c              3   X   K   | ]"  }|j                         j                          $ y wrj   rR  r2  r  r.  s     r*   r  z'ModuleTest.test_cuda.<locals>.<genexpr>       >eQRquuw{{}>e   (*c              3   X   K   | ]"  }|j                         j                          $ y wrj   rR  rS  s     r*   r  z'ModuleTest.test_cuda.<locals>.<genexpr>  rT  rU  c              3   <   K   | ]  }|j                           y wrj   rR  rS  s     r*   r  z'ModuleTest.test_cuda.<locals>.<genexpr>       8YQ8Yr  c              3   <   K   | ]  }|j                           y wrj   rX  rS  s     r*   r  z'ModuleTest.test_cuda.<locals>.<genexpr>  rY  r  )retain_graph)+r   r  unittestSkipTestr   rJ   r  r8   rb   floatr  r  rC  r   r   rD   cudar  r  r  copy_r  r  r  r$   r&  r   r  rM  rD  r   r  r^   r  
randn_liker  r   autogradr  r  rR  r2  r'  )rl   r  	cpu_inputrO  cpu_input_tupleis_any_input_complexgpu_input_tuple
cpu_module
gpu_module	cpu_param	gpu_paramcpu_pgpu_p
cpu_output
gpu_outputr@   cpu_gradOutputgpu_gradOutputcpu_gradInputgpu_gradInputcpu_d_pgpu_d_pcpu_gradInputsgpu_gradInputscpu_d_igpu_d_ioutputs_cpuoutputs_gpucpu_gggpu_ggs                                 r*   rQ   zModuleTest.test_cuda  s8    5 5##$>??t112 W	H)Iekk2H+5i+Gii\O#&&scr&s#s $_xHO)))4+@+@AJ)))4+@+@AGGINNPJ!11*=I!11*=I #IaL)A, ? (u

  '( &&7&&7++J7++J7"++JHJ"++JHJz#3U;']
']
!!*jt~~TUch!i 1X Y!+!1!1!3!;!;!=!/!7!7
!C ) 3 3JQ[]k l ) 3 3JQ[]k l%%m]^_mr%s(+IaL)A,(G Y$GW))'7VW)XYY ""4+Q+Q'9
'9
:'7?!+AJ!+AJ!&!1!1*D!Q!/!7!7
!C!J!J!L/3,!&!4!4#eJ,A,A,C&DD"!%	 "5 "'
 "'!4!4#eJ,A,A,C&DD"!%	 "5 "' ),NN(K l$GW))'7VWej)kl (",.."2"6"6"83>eVd>e;e"eK",.."2"6"6"83>eVd>e;e"eK",.."2S8Y.8Y5Y"YK",.."2S8Y.8Y5Y"YK,,#~&77%
@U@U@W:XX!% - ' ,,#~&77%
@U@U@W:XX!% - ' %%m]^_mr%s(+FF(; l$GW))'7VWej)kl 	:GoW	H W	H W	Hs   T!U''U0)rq   rr   rs   r   r   ru   rv   r  r  r  r4  r'  rQ   __classcell__r"  s   @r*   r  r  G  sR    "# "ryy " " " ";46*#UJ[Hr,   r  c                       e Zd Zd Zy)InputVariableMixinc                 J    t         j                  | d      }fd |      S )NFc                     t        | t        j                        r)| j                         s| j	                         rd| _        | S  t        |       fd| D              S )NTc              3   .   K   | ]  } |        y wrj   rk   )r  r  map_variabless     r*   r  zGInputVariableMixin._get_input.<locals>.map_variables.<locals>.<genexpr>  s     At}T2A   )r  r8   r  is_floating_pointrM  r   r  )r>   r  s    r*   r  z4InputVariableMixin._get_input.<locals>.map_variables  sH    !U\\*&&(ALLN&*AOtAwAqAAAr,   )r  r  )rl   r  r  s     @r*   r  zInputVariableMixin._get_input  s(    ##D%0	B U##r,   N)rq   rr   rs   r  rk   r,   r*   r  r    s    $r,   r  c                   @     e Zd Z fdZd Zd Zd Zed        Z xZ	S )NewModuleTestc                 z   t        |   |i | |j                  dd      | _        |j                  dd      | _        |j                  dd      | _        |j                  dd      | _        |j                  dd      | _        |j                  dd      | _        |j                  d	d
      | _	        |j                  dd      | _
        |j                  dd      | _        |j                  dd      | _        |j                  dd       | _        |j                  dd      | _        |j                  dd      | _        y )Nr  Fcheck_inplacer^   Tskip_doubler  rH   rI   r\   test_cpur1  r_   r  r   supports_fwgrad_bwgrad)r  r  r  r  r  r^   r  r  rH   rI   r  r1  r_   r  r   r  r!  s      r*   r  zNewModuleTest.__init__  s   $)&)ZZ/
#ZZ?$jj)94@!::mU;K7K7$jj)95A

:t4$*JJ/Eu$M!"(**-A4"H#)::.CT#J #)::.CU#K &,jj1I5&Q#r,   c           	         t        d j                         D              }t        |      fd}| j                  r6dk(  sJ t	        j
                  |d         }j                  |d   |       n@j                  t        |||z   | j                  | j                  | j                               | j                  rAj                  t        |||z   | j                  | j                  | j                               y y )Nc              3       K   | ]  }|  y wrj   rk   rS  s     r*   r  z1NewModuleTest._check_gradients.<locals>.<genexpr>/  s     6Qq6r  c                  6    |rJ j                  | d        S rj   )r  )inputs_and_paramsro   r  
num_inputsr  s     r*   fn_to_gradcheckz7NewModuleTest._check_gradients.<locals>.fn_to_gradcheck2  s&    :%%f.?.LMMr,   r6   r   )r_   	fast_modecheck_forward_ad)r_   r  check_fwd_over_rev)r  r  r  r1  r8   r  r  
assertTruer   r_   r  r   r^   r   r  )rl   r  r  input_tupleparamsr  test_input_jacobianr  s    ``    @r*   _check_gradientszNewModuleTest._check_gradients.  s    6&"3"3"566%
	N $$?"?"'"9"9+a."I$$V[^=PQ  ?K&<P>B>U>U595M5M<@<T<T"V W
   f@TBFBYBY9=9Q9QBFB]B]"_ ` r,   c                    t        j                         }t        j                  d       t        |t              r|n|f}| j                  |       j                          | j                  rt        |      dk(  sJ |d   } | j                  | j                  ddi}|j                  }t               5   |      }d d d        j                  |j                  |       t        |      }	|	j                         }
t               5   ||
      }d d d        j!                  |
j                  |       j                         |j"                  j                         j%                         }|j&                  7t        j(                         5  |j&                  j+                          d d d        |	j&                  7t        j(                         5  |	j&                  j+                          d d d        |j-                  |       |j-                  |       j                  |j&                  |	j&                         dfd	}t/        d |D              rt0        rt	        d |D              }j3                         j5                           |   |t         j4                  j6                  d       t         j4                  j9                         dkD  rt	        d |D              }j5                  d       t         j4                  j;                  d      5   |  d d d         |t         j4                  j6                  d       n@d	 fd
fdfdt	        fd|D              }j3                           |   |t         j6                         t	        fd|D              }j=                           |   |t         j>                         t0        r| j@                  rt	        fd|D              }j3                         j5                           |   |t         j4                  j6                  d       t	        d |D              }jC                           |   |t         j6                         t	        d |D              }j5                           |   |t         j4                  j6                  d       | jD                  rYt         jF                  jD                  jI                  d      5   |   |t         j4                  j6                  d       d d d        t         j4                  j9                         dk\  rqt	        d |D              }j5                  d       t         j4                  j;                  d      5   |  d d d         |t         j4                  j6                  d       | jJ                  sXt	        fd|D              }j=                         j5                           |   |t         j4                  j>                  d       | jL                  sXt	        fd|D              }jO                         j5                           |   |t         j4                  jP                  d       t        j                  |       y # 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   WxY w)Nr6   r   inplaceTc                     j                         D ]7  }j                  ||        |j                  |j                         |       9 y rj   )r  assertIsInstancer&  
get_device)tensor_type	device_idr?   r  r  s      r*   assert_module_parameters_arez<NewModuleTest._do_test.<locals>.assert_module_parameters_areu  sI    &&( E**1k:())!,,.)DEr,   c              3   P   K   | ]  }t        |t        j                           y wrj   )r  r8   
LongTensorrN  s     r*   r  z)NewModuleTest._do_test.<locals>.<genexpr>{  s     D1z!U--.Ds   $&c              3   <   K   | ]  }|j                           y wrj   r_  rN  s     r*   r  z)NewModuleTest._do_test.<locals>.<genexpr>~  s     >Q>r  c              3   >   K   | ]  }|j                  d         ywr  r  rN  s     r*   r  z)NewModuleTest._do_test.<locals>.<genexpr>  s     #C!AFF1I#C   c                     | j                         r| j                  |      S | j                         r| j                  |      S | S rj   )rM  r   r  )r  realr  s      r*   to_typez'NewModuleTest._do_test.<locals>.to_type  s<    $$&!99W----/!99T?*!Mr,   c                 4     | t         j                  d       S rj   )r8   float16r.  r  s    r*   to_halfz'NewModuleTest._do_test.<locals>.to_half  s    q%--66r,   c                 P     | t         j                  t         j                        S rj   )r8   float32	complex64r  s    r*   	to_singlez)NewModuleTest._do_test.<locals>.to_single  s    q%--AAr,   c                 P     | t         j                  t         j                        S rj   )r8   float64r  r  s    r*   	to_doublez)NewModuleTest._do_test.<locals>.to_double  s    q%--1A1ABBr,   c              3   .   K   | ]  } |        y wrj   rk   r  r:   r  s     r*   r  z)NewModuleTest._do_test.<locals>.<genexpr>       B	!Br  c              3   .   K   | ]  } |        y wrj   rk   r  r:   r  s     r*   r  z)NewModuleTest._do_test.<locals>.<genexpr>  r  r  c              3   J   K   | ]  } |      j                           y wrj   r  r  s     r*   r  z)NewModuleTest._do_test.<locals>.<genexpr>  s     #MAIaL$5$5$7#M    #c              3   <   K   | ]  }|j                           y wrj   )cpurN  s     r*   r  z)NewModuleTest._do_test.<locals>.<genexpr>  s     #AAEEG#Ar  c              3   <   K   | ]  }|j                           y wrj   r  rN  s     r*   r  z)NewModuleTest._do_test.<locals>.<genexpr>  s     #BAFFH#Br  F)enabledrP   c              3   >   K   | ]  }|j                  d         ywr  r  rN  s     r*   r  z)NewModuleTest._do_test.<locals>.<genexpr>  s     'Gaq	'Gr  c              3   J   K   | ]  } |      j                           y wrj   r  r  s     r*   r  z)NewModuleTest._do_test.<locals>.<genexpr>  s     'Q	!(9(9(;'Qr  c              3   J   K   | ]  } |      j                           y wrj   r  )r  r:   r  s     r*   r  z)NewModuleTest._do_test.<locals>.<genexpr>  s     'Oa
(9'Or  rj   ))r8   get_num_threadsset_num_threadsr  r  r  __repr__r  r  r   rD   _versionr   r&  r   rM  assertNotEqualr  rD  r  no_gradr  backwardallr   r^  r_  FloatTensordevice_countrH  rb   DoubleTensorr  r  r  backendsflagsr  r  half
HalfTensor)rl   r  r  r  num_threadsr  	module_ipinput_versionr  input_ipinput_ip_clone	output_ipr  r  r  r  r  r  s    ``           @@@@r*   r  zNewModuleTest._do_testJ  sU   ++-a )%7eeXi= 	 {#q(((NE((($*?*?NNI!NNM!# ''!!%..-@H%^^-N!# 6%n5	6$$^%<%<mL!!&)4;;$$&..0Dzz%]]_ 'JJ$$&'}}(]]_ *MM'')*OOD!t$!!%**hmm<	E DDD  >+>>KLLN!K ()?)?Czz&&(1,##C{#CCAZZ&&q) )K(),UZZ-C-CQG"7BC  BkBBKLLNK ():):;  BkBBKMMOK (););<T22
 $#M#MM##%$,UZZ-C-CQG $#A[#AA

$,U->->? $#Bk#BB$,UZZ-C-CQG ::--33E3B P,4UZZ5K5KQOP ::**,1 #('G;'G"GKKKN**1- -,-01G1GK''"''Q['Q"QKMMO((*K(01H1H!L ~~"''O;'O"OKKKM&&(K(01F1FJk*w' '6 6' '* *.) )vP P- -sT   <	Z#	Z0Z=[
['[$[1#Z-0Z:=[
[[!$[.1[;c                 &    | j                  dd      S NrJ  Fr  r  s    r*   _get_targetzNewModuleTest._get_target      }}Xu--r,   c                 &    | j                  dd      S NrD   Fr  r  s    r*   rD   zNewModuleTest.constructor_args      }}/77r,   )
rq   rr   rs   r  r  r  r  r  rD   r}  r~  s   @r*   r  r    s/    R `8Q+f. 8 8r,   r  c                        e Zd Zej                  j                  dh      Z fdZd ZddZd Z	e
d        Ze
d        Z xZS )	CriterionTestrJ  c                 j   t        |   |i | |j                  dd      | _        |j                  dd      | _        |j                  dd      | _        |j                  dd      | _        |j                  dd      | _        |j                  dd      | _        |j                  d	d      | _	        |j                  d
d      | _
        |j                  dd      | _        |j                  dd      | _        |j                  dd       | _        | j                  t        j                         | _        y y )NrQ   Tr  Fr^   r  check_bfloat16check_complexr  rH   rI   r\   r_   rJ   )r  r  r  r  r  r^   r  r  r  r  rH   rI   r_   rJ   r8   r   r!  s      r*   r  zCriterionTest.__init__  s
   $)&) &

; ="(**-A5"I$jj)94@ **\48$jj)95A#ZZ?

:t4K6$jj)95A"(**-A4"H#ZZ>%!&!8!8!:D &r,   c                 (  
 t        | j                        5   | j                  | j                   
| j	                         }
j                          t        
       | j                         }| j                  g|j                  
||| j                        }t        |      t        |      f| j                  z   
fz   } | j                  | }|j                  ||       | j                  r
	 d d d        y t        d 
j                         D              }t!        |t              s|f|z   |fz   }
fd}	n||z   |fz   }
fd}	t#        |	|| j$                         | j&                  rt)        |	|| j$                         d d d        y # 1 sw Y   y xY w)Nr  c              3       K   | ]  }|  y wrj   rk   rS  s     r*   r  z)CriterionTest.__call__.<locals>.<genexpr>  s     :1:r  c                      | |      S rj   rk   )r  rJ  r  r  s      r*   apply_fnz(CriterionTest.__call__.<locals>.apply_fn  s    !%00r,   c                      | ||      S rj   rk   )r6  r7  rJ  r  r  s       r*   r  z(CriterionTest.__call__.<locals>.apply_fn  s    !&&&99r,   )r_   )r   rJ   r   rD   r  r  r{  r  rG   _forward_criterionr  r   r&  r  r  r  r  r   r_   r^   r   )rl   r  r  rJ  r-  ref_argsr0  r  inputsr  r  s             @r*   r  zCriterionTest.__call__  s~   t112 "	\%T%%t'<'<=FOO%E OOK%%'F  ,2265&UYUdUd2e$UOXf-=>PTZS\\0t00(;%%c<8&&#"	\ "	\& :f&7&7&9::FeU+F*fY61 6)3: h4;R;RS""h4CZCZ[E"	\ "	\ "	\s   CF5B
FFc                    d
fd	t         r| j                  st        j                  d      t	        | j
                        5  | j                         }| j                         } | j                  | j                   } | j                  | j                   } ||d      }|j                         s|j                         r	 ||      }|j                  |       |j                  |       t        |      }t        |      }	|j                          |t        j                   t        j"                  hv r9| j                         }| j                         } | j                  | j                   }|j%                  ||||      }
|j%                  |||	|      }|j'                  |
||t        j                   t        j"                  hv rdnddd	       |j)                  |||
||      }|j)                  ||||	|      }|j'                  |||t        j                   t        j"                  hv rdnddd	       d d d        y # 1 sw Y   y xY w)NFc                     t        | t        j                        r/| j                         j	                        j                        S t        | t              rt        fd| D              S | S )Nr   c              3   2   K   | ]  } |        y wrj   rk   )r  r5  convert_dtyper   r   s     r*   r  zACriterionTest.test_cuda.<locals>.convert_dtype.<locals>.<genexpr>$  s     Q]1e]CQs   )r  r8   r  r  r   requires_grad_r  )r9  r   r   r  s    ``r*   r  z.CriterionTest.test_cuda.<locals>.convert_dtype   sS    #u||,zz|U3BB=QQC'QSQQQ
r,   rK  Tr  rR   g-C6:?r   rP  r  )r   r  r\  r]  r   rJ   r  r  r   rD   r  rM  r  r   r_  r8   r  bfloat16r  r&  _backward_criterion)rl   r  r   r  rc  
cpu_targetrg  rh  	gpu_input
gpu_targetrm  rn  rq  rr  r  s                 @r*   rQ   zCriterionTest.test_cuda  s7   	  5 5##$>??t112 %	{)I))+J)))4+@+@AJ)))4+@+@AJ &i=I++-1F1F1H*:u=
OOE"OOE" y)I
+JOO U^^44 OO-	!--/
-T--t/D/DE
"55j)Zdn5oJ"55j)Zdn5oJ!!*j/4U^^8T/TtZ^efty " { &99Iz:* : VM%99Iz:* : VM !!-/4U^^8T/TtZ^efty " {I%	{ %	{ %	{s   G3I  I	c                 &    | j                  dd      S r  r  r  s    r*   r  zCriterionTest._get_targetR  r  r,   c                 &    | j                  dd      S r  r  r  s    r*   rD   zCriterionTest.constructor_argsU  r  r,   c                 &    | j                  dd      S )Nr  Fr  r  s    r*   r  zCriterionTest.extra_argsY  s    }}\511r,   rj   )rq   rr   rs   r  r  unionr  r  rQ   r  r  rD   r  r}  r~  s   @r*   r  r    s^     #66<<hZH; #\J1{f. 8 8 2 2r,   r  c                    t        j                  |t         j                  |d      }|Et        j                  |t         j                  |      |z  j                         j                         } ||      }t        j                  ||      }|j                  |       |j	                         }	|j                         j	                         j                         }
|j	                         } |	|
      }|j                  |       | j                  ||||d       | j                  |j                  j                  |
j                  j                  ||d       y )NT)r   rH  r   rG  )rH  FrP  )r8   r   r  r   r  r^  r  ra  r  r  r&  r  r  )r  oprH  inp_dimsprecr:  r6  out1grad_input1op_bfp16r7  grad_input2out2s                r*   _test_bfloat16_opsr  ^  s
   [[vUYZF**XU^^FKlZaacrrtf:D""47KMM+ {{}H]]_%%'668F&&(KFDMM+$4dN&++**FKK,<,<4d`efr,   c                    |s|j                  d        ||      }|s&t        j                  |      }|j                  |       |r.| j	                  |j                         |j                                |s|j                         D ]H  }|j                  s| j	                  |j                  t        j                  |j                               J | j	                  |j                  t        j                  |             y y )NT)
r  r8   	rand_liker  r&  rc   r  r   r  r  )r  r  r  
check_size	inferencer-  gOr?   s           r*   _test_module_empty_inputr  q  s    4 
+C__S!Rchhj#((*5""$ 	HA%%affe.>.>qvv.FG	H 	chh(8(8(=>	 r,   c                       G d dt         j                         G fddt         j                        }         } |        }t        j                  ||      }|||fS )Nc                         e Zd Zd fdZ xZS ) _create_basic_net.<locals>.Layerc                     t         |           t        j                  t	        j
                  dd            | _        t        j                  t	        j                  dddd            | _	        y )NrW   rS   r6   r  )
r  r  ru   r  r8   rn  layer_dummy_paramBufferr  layer_dummy_buf)rl   r"  s    r*   r  z)_create_basic_net.<locals>.Layer.__init__  sJ    G%'\\%++a2C%DD"#%99U[[Aq!-D#ED r,   r  Nrq   rr   rs   r  r}  r~  s   @r*   Layerr    s    	F 	Fr,   r  c                   $     e Zd Zd fdZ xZS )_create_basic_net.<locals>.Netc                     t         |                   | _        t        j                  t        j                  dd            | _        t        j                  t        j                  dddd            | _
        y )NrW   rS   r  r6   )r  r  l1ru   r  r8   rn  dummy_paramr  r  	dummy_buf)rl   r  r"  s    r*   r  z'_create_basic_net.<locals>.Net.__init__  sR    GgDG!||EKK1,=>DYYu{{1aA'>?DNr,   r  r  )r"  r  s   @r*   Netr    s    	@ 	@r,   r  )ru   rv   
Sequential)r  lr   r	  r  s       @r*   _create_basic_netr!    sR    F		 F@bii @ 	AA
aAa7Nr,   )r  F)Nr  )Nr  r  )Nr"  r  r  )r  rV  )r  )rV  r  )r6   r6   Nr  )r   r  )rV  rP   r  Fr  )rk   r  Nr?  )abcr   r(  r\  copyr   	functoolsr   r   	itertoolsr   operatorr   r8   
torch.cudatorch.nnru   torch.nn.functional
functionalr~   r	   r%   $torch.testing._internal.common_utilsr
   r   r   r   r   r   r   r   #torch.testing._internal.common_cudar   r   torch.autograd.gradcheckr   r   torch.autogradr   torch.typesr   torch.backends.cudnntypingr   r   r   r   r   r   r   r)  r  r+   r0   r   rb   module_testsre   rw   r   r   r   r   r   r   r   r   r   r  r
  r  r  r$  r5  r<  rC  rU  rZ  rr  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r"  r(  r5  r>  rF  rT  rZ  rh  rn  rt  r{  r  r  r  interpolatesoftmaxr  r  gelureluSizenew_module_testsr  padding_modecpp_padding_moder  r  r  r  joinmapr{  cpp_paddingrF   r  r  non_linear_activations_no_batchr  __annotations__non_linear_activationactivation_test_infor  
extra_infoupdater  r  r  r   r  r  r  r  r  r!  r%  r'  r+  r/  r8  rC  rE  r]  r   criterion_testsrj  regression_criterion_no_batch
reductionsro  r!   regression_test_info!classification_criterion_no_batchr  classification_cpp_parityr   r{  classification_test_infor  r  r  r  r  r  r  r  r!  rk   r,   r*   <module>rL     s(       %        -C C C F K # (  D D D&&	'| 	 >Zll	 	'J:ll
 	ll	 	#Nll 	6ll	 	#+] ll	Y6x6
$$ $(&$$$$ "$ $ "$ $$$ $ $"$$$,$.$.$"$$$,$"$$$,$"$"$"$"$""$$$$$$$"$"$"$"$*$$$$$$$$$($,6"J!#JJ #$J '(	J
 %&J "#J *+J ,-J )*J J $%J -.J ()J /0J J  "#!J" !"#J$ %J& "#'J( )J* ()+J, #$-J. 01/J0 451J2 3J4 %&5J6 *+7J8 9J: %&;J< *+=J>  !?J@ '(AJB CJD  !EJF GJH +,IJJ +,KJL ()MJN ()OJP &'QJR -.SJT "#UJV ,-WJX 45YJZ #$[J\ &']J^ 67_J` %&aJb *+cJd +,eJf 	"@ll	gJz 	%Jll
{JP 	(Ull
QJf 	(Ull
gJ| 	(Ull
}JR 	(Ull
SJh 	"@	iJ| 	!FLll}JN 	 DJll	OJb 	#?Wll	cJv 	">Vll	wJJ 	#>Vll	KJ^ 	*Jbll	_Jr 	"ppll	sJF 	%5allGJ^ 	%7lll_Jv 	)nUll
wJL 	'E#ll
MJb 	/T#llcJz 	7d#ll{JR 	?u#llSJj 	:[#llkJD	 	'E#	E	JX	 	 =O#ll
Y	Jn	 	%=O#ll	o	JB
 	#D\ll	C
JV
 	"C[ll	W
Jj
 	*Ogll	k
J~
 	%5U#ll
JV 	%G5  #ll!WJz 	%?k#ll{JT 	)FX#ll
UJj 	#=OllkJ| 	3=Oll}JN 	+L^llOJ` 	*M_llaJr 	+N`llsJD 	*H"#ll
EJZ 	=["#ll[Jt 	=["$lluJN 	%J"#llOJf 	(U"#llgJ~ 	*H"#	JR 	 DJ"#ll
SJh 	!FL"lliJz 	)PV"ll{JL 	#G_"ll	MJ` 	"F^"ll	aJt 	*Rj"ll	uJH 	%*Q"ll	IJ\ 	%Gt"ll]Jt 	&.U"lluJB 	&.U.llCJT 	&.U^UJd 	@Gll#$VWeJv 	@Xll#$VW	wJJ 	"CGllKJ\ 	"CXll]Jn 	"7uGll	oJB 	"7uGll	CJV 	0@RLllWJf 	/XDLllgJx 	/XDLllyJJ 	&R B7!KJZ 	O~7#![Jj 	"@llkJx 	$B!llyJF 	#AMMT]^r)llGJX 	#AMMT]^r2YJh 	#AMMTXabr/lliJz 	#AMM2T]^q/ll{JL 	#AMMT\lqr5
 (llMJd 	#AMMDW_otu5
 .lleJ| 	#AMM2T\lqr5
 .ll}JT 	#AMMT\lqr5
 1
UJj 	#AMMT\lpq4
 6llkJB 	#AMM2T\lpq4
 <llCJZ 	#AMMS\]6 "8ll
[Jp 	#AMMT]^6  )ll
qJF 	#AMMtZcd6  /ll
GJ\ 	#AMM2T]^6  /ll
]Jr 	#AMMT]^6  2	sJF 	#AMMT^nst5
  *llGJ^ 	#AMMT^nst5
  3
_Jt 	#AMMT)35J5
  0lluJN 	#AMM2)35J5
  0llOJh 	#AMM8)35J5
  =lliJB 	#AMM8)35J5
  =llCJ\ 	#AMMTXbrvw4
  >ll]Jt 	#AMM8)34I4
  KlluJN 	#AMMT]mrs5
  )llOJf 	#AMMT]mrs5
  2
gJ| 	#AMMT)2%I5
  /ll}JV  	#AMM2T]mrs5
  /llW Jn  	#AMM8)2%I5
  <llo JH! 	#AMM8)2%I5
  <llI!Jb! 	#AMMTXaquv4
  =llc!Jz! 	#AMM8)2$H4
  Jll{!JT" 	#AMMT]^6 #)ll
U"Jj" 	#AMMT]^6 #2	k"J~" 	#AMMSW^gh6 #/ll
"JT# 	#AMM2T]^6 #/ll
U#Jj# 	#AMMT_otu5
 #+llk#JB$ 	#AMMT_otu5
 #4
C$JX$ 	#AMM	15KW\^5
 #1llY$Jr$ 	#AMM2T_otu5
 #1lls$JN% 	#AMM	PT)4DJ4
 #?llO%Jh% 	#AMM2T_ost4
 #?lli%JD& 	#AII264"llE&JT& 	#AII1EMMJJ(llU&Jf& 	#AII153!*llg&Jv& 	#AII153"llw&JF' 	#AII1EMMJJ(llG'JX' 	#AII153*llY'Jj' 	#AII153*llk'J|' 	#AII264,}'JL( 	#AMMr:7&llM(J\( 	#AMMq96!.ll](Jl( 	#AMMq96&llm(J|( 	#AMMq96#ll}(JL) 	#AMMq96#llM)J\) 	#AMMq96%])Jj) 	Eollk)J|) 	Kull})JN* 	*Ku%ll	O*Jb* 	#1[llc*Jt* 	!2\llu*JF+ 	.2\%ll	G+JZ+ 	#N"ll[+Jl+ 	&Allm+Jv+ 	&All	w+JB, 	&@ll	C,JN, 	&+cA$llO,J^, 	&9.ll_,Jl, 	-(1  !llm,JN- 	-Q3/= *tllO-Jj- 	-'1 Dllk-JD. 	-Q3/= DllE.J`. 	!Q1aaff5= V*tll!a.JD/ 	=&Vll
E/JZ/ 	T!.ll[/Jl/ 	Fjejj!Q01.llm/J~/ 	&e4l" )
/J Z0 '*5U'W #
"L"  
6li/ aQ(DIIc#w&7883>dQh&
u$<G$<<<"1#QK"#Q1gq!T<!P)8 <22= ?7 8H6H&M &'$~]3##ll!	

#
L# 6
F6$h/6 e%RWR^R^_6 EPUP\P\]	6 55<<@6 
OU\\*6 _ell+6 
OU\\*6 ?ELL16 %,,/6 /5<<06 ?ELL16 _ell+6  ou||,!6" ou||,#6$ _ell+%6& _ell+'6( ell+. %,,/"ELL1 %,,/ell+"ELL156 !4T	? 8 = 
2).! 3667LbQJ
+01
2 BF"(4 TZ=@, Z^RU%IP [a14
  @D &, 	 4.  ]b*0 "L %#$7DI $*$:6.0844 4- D) & D!  %
<jI 
1OD)6	{3'+S8!ll /0
1  
1I*9+6=/*8!ll /0
1" 	AI
 DFpqEGklGIlmACghPRijX3 DFde 	X2
  !KMcd1% !6 -u5A ,d39o 
   %
.56W6@/B 5*T8Y#6	{3'+S"'$)8 ,00t<  >AA$KJ##J/345 x> x>vJ" J"ZDH DHN$ $E8&
 E8Pv2& v2rg&? r,   