
    sgrI             %          U d dl Z d dlZd dlZd dlZd dlmZ d dlmZ d dlm	Z	m
Z
mZmZmZ d dlZd dlmZ d dlmZ d dlmZmZmZmZmZmZmZmZmZmZmZmZmZm Z m!Z! d dl"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z( d d	l)m*Z*m+Z+ d d
l,m-Z- d dl.m/Z/m0Z0m1Z1m2Z2m3Z3 d dl4m5Z5  G d d      Z6 G d de      Z7 G d d      Z8 G d d      Z9 G d de/      Z:d Z;ddZ<d Z=ddZ>d Z?ddZ@d ZAddZBd ZCdd ZDd! ZEdd"ZFd# ZGdd$ZHd% ZIdd&ZJd' ZKdd(ZLd) ZMdd*ZNd+ ZOdd,ZPd- ZQdd.ZRd/ ZSdd0ZTd1 ZUdd2ZVd3 ZWd4eeXej                  f   d5eXfd6ZZ	 dd5ee6   fd8Z[ e9ee<e=d9d: e- e1d;      d<d=ej                  d>k  xr ej                  d?kD  @       e- e1dA      d<dB       e- e1dC      d<dD       e- e+ej                   e*dEdFG      i      dHdI       e- e1dJ      d<dK      fL       e9ee>e?dMdMdN e- ej                  dO      dHdIdPdQ R       e- e1dO      d<dSdPT       e- e1dO      d<dUdPT       e- e1dO      d<dV       e- e1dO      d<dW       e- e1dO      d<dXdPT       e- e1dO      d<dY       e- e1dO      d<dZdPT       e- e1dO      d<d[dPT       e- e1dO      d<d\       e- e1dO      d<d]dPT       e- e1d^      d<dB       e- e1dO      d<d_dPT       e- e1dO      d<d`dPT       e- ej                  dO      dHdIdaT       e- e1dO      d<dSdaT       e- e1dO      d<dUdaT       e- e1dO      d<dXdaT       e- e1dO      d<dZdaT       e- e1dO      d<d[daT       e- e1dO      d<d]daT       e- e1dO      d<d_daT       e- e1dO      d<d`daT      fb       e9ee@eAdcddded:dfd d dgdh di gf e- e+ej                   e*djdjk      ej                   e*djdjk      i      d<dl      f e-e0d<dmdn @       e- e1d;      d<d=ej                  d>k  xr ej                  d?kD  @       e- e1dA      d<dB       e- e1dC      d<dD       e- e1dJ      d<dK      fo
       e9eeBdp gdq gdr ds gdt du gdv gdw gdx dy gfeCdcd:dddz e- e+ej                   e*d{d|k      i      d<dld} @       e- e+ej                   e*djdjk      ej                   e*djdjk      i      d<dl       e- e+ej                   e*d~dk      i      dd      f e-e0d<dmd @       e- e1d;      d<d=ej                  d>k  xr ej                  d?kD  @       e- e1d      d<dB       e- e1dC      d<dD       e- e1dJ      d<dK      f
       e9eeDeEd9d: e-e0d<dmd @       e- e1d;      d<d=ej                  d>k  xr ej                  d?kD  @       e- e1dA      d<dB       e- e1dC      d<dD       e- ej                  d      d<dV       e- e1dJ      d<dK      fL       e9eeFeGdcdddzd: e- e+ej                   e*d{d|k      i      d<dld @       e- e+ej                   e*djdjk      ej                   e*djdjk      i      d<dl       e- e+ej                   e*d~dk      i      dd      f e-e0d<dmd @       e- e1d;      d<d=ej                  d>k  xr ej                  d?kD  @       e- e1d      d<dB       e- e1dC      d<dD       e- e1dJ      d<dK      f	       e9eeHeId9d: e- e1d;      d<d=ej                  d>k  xr ej                  d?kD  @       e- e1d      d<dB       e- e1dC      d<dD       e- e+ej                   e*ddk      i      d<d`       e- e1dJ      d<dK       e- ej                  d      d<d`      fL       e9eeJeKd7d:dNdN e-e0d<dS       e- e+ej                   e*ddFG      i      d<dD       e- ej                  d      d<dZ       e- ej                  d      d<d[       e- ej                  d      d<d       e- ej                  d      d<d       e- e1d;      d<d=ej                  d>k  xr ej                  d?kD  @       e-ej                  dHdId @      f       e9eeLeMd9d: e-e0d<dmd @       e- e1d;      d<d=ej                  d>k  xr ej                  d?kD  @       e- e1d      d<dB       e- e1dC      d<dD       e- e1d      d<dX       e- e1dJ      d<dK      fL       e9eeNeOd9d: e- e1d;      d<d=ej                  d>k  xr ej                  d?kD  @       e- e1d      d<dB       e- e1dC      d<dD       e- e+ej                   e*ddk      i      d<dW       e- e1dJ      d<dK      fL       e9eePeQd9d: e-e0d<dmd @       e- e1d;      d<d=ej                  d>k  xr ej                  d?kD  @       e- e1dA      d<dB       e- e1dC      d<dD       e- e+ej                   e*ddk      i      d<dYe3@       e- e1dJ      d<dK      fL       e9eeReSd9d: e-e0d<dmd @       e- e1d;      d<d=ej                  d>k  xr ej                  d?kD  @       e- e1dA      d<dB       e- e1dC      d<dD       e- e1dJ      d<dK      fL       e9e eTd gd gd d gd d d gd gd gd d gfeUdcddd:ddNd dNd dd gfdz e- e1d;      d<d=ej                  d>k  xr ej                  d?kD  @       e- e1d      d<dB       e- e1dC      d<dD       e- e+ej                   e*ddk      i      d<dYe3@       e- e1dJ      d<dK      f
       e9e!eVeWd7d:ddig fdN e-e0d<       e- e2d             e- e1d      d<dZ       e- e1d      d<d=       e- ej                  d      d<dS       e- e1d      d<dX       e- e1d      d<dm       e- e1d      d<d       e- e1d      d<d       e- e1d      d<d       e- e1d      d<dU      f      gZdee9   eed<    G d d      Zfy)    N)deepcopy)Enum)AnyDictListTupleUnion)Tensor)	Parameter)Adadelta	AdafactorAdagradAdamAdamaxAdamWASGDLBFGSNAdam	OptimizerRAdamRMSpropRpropSGD
SparseAdam)
ConstantLRExponentialLRLinearLRPolynomialLRReduceLROnPlateauStepLR)toltoleranceOverride)DecorateInfo)_TestParametrizer	skipIfMpsskipIfTorchDynamo	skipIfXpuTEST_WITH_TORCHDYNAMO)&_get_foreach_kernels_supported_devicesc                   b    e Zd ZdZg dZ	 d	deee   ee   e	e
e
f   f   de	ee
f   defdZd Zy)
OptimizerInputz@Contains args / kwargs to be passed to an optimizer constructor.paramskwargsdescr-   r.   r/   c                 .    || _         || _        || _        y Nr,   )selfr-   r.   r/   s       \/var/www/html/venv/lib/python3.12/site-packages/torch/testing/_internal/common_optimizers.py__init__zOptimizerInput.__init__8   s     	    c                 T    d| j                    d| j                   d| j                   S )Nzparams=z	, kwargs=z, desc=r,   r2   s    r3   __repr__zOptimizerInput.__repr__C   s'    Yt{{m7499+NNr5   N) )__name__
__module____qualname____doc__	__slots__r	   r   r   r
   r   r   strr4   r8    r5   r3   r+   r+   3   sZ    J,I 		d9otF|T#s(^CD	 S#X	 		Or5   r+   c                       e Zd ZdZdZdZy)OptimizerErrorEnumz;Enumerates when an error is raised when testing optimizers.r      N)r:   r;   r<   r=   CONSTRUCTION_ERROR
STEP_ERRORr@   r5   r3   rB   rB   G   s    EJr5   rB   c                   <    e Zd ZdZg dZej                  edddZy)ErrorOptimizerInputz
    An OptimizerInput that will cause the optimizer to throw an error when constructed.
    Includes the type and string of the resulting error.
    optimizer_error_inputerror_on
error_typeerror_regexr9   )rJ   rK   rL   c                <    || _         || _        || _        || _        y r1   rH   )r2   rI   rJ   rK   rL   s        r3   r4   zErrorOptimizerInput.__init__V   s"     &;" $&r5   N)	r:   r;   r<   r=   r>   rB   rD   RuntimeErrorr4   r@   r5   r3   rG   rG   N   s%    
 SI $66'r5   rG   c                       e Zd ZdZd d gfdddddi g fdddddd	d	dd
dedeedf   deedf   dedededededededeedf   fdZd Z	e
d        Zy	)OptimizerInfoz,Optimizer information to be used in testing.c                     t        | dd      S N?
   gamma	step_sizer    opts    r3   <lambda>zOptimizerInfo.<lambda>v       F3cR@ r5   c                     t        |       S r1   r   rY   s    r3   r[   zOptimizerInfo.<lambda>w       -c2 r5   foreachdifferentiable)ra   rb   maximize
capturableFTr@   N)scheduler_inputssupported_implsnot_og_supported_flagssupports_sparsehas_capturable_argonly_supports_sparse_gradsmetadata_for_sparsesupports_complexstep_requires_closuresupports_param_groupssupports_multiple_devicesskips
decoratorsoptim_error_inputs_funcsupports_fused_on	optim_clsrf   .rg   rh   ri   rj   rl   rm   rn   ro   rs   c                    || _         || _        || _        || _        || _        || _        || _        |	| _        || _        |
| _	        || _
        || _        || _        g |r|ng |r|ng | _        || _        || _        y r1   )rt   optim_inputs_funcre   rf   rg   rh   ri   rk   rj   rl   rm   rn   ro   rq   rr   rs   )r2   rt   rv   re   rf   rg   rh   ri   rj   rk   rl   rm   rn   ro   rp   rq   rr   rs   s                     r3   r4   zOptimizerInfo.__init__g   s    j #!2 0.&<#."4#6 *D' 0%:"%:")B&
&jB
e"
 (?$!2r5   c                     g }| j                   D ]U  }t        |t              r2|j                  |||||      s)|j	                  |j                          E|j                  |       W |S r1   )rq   
isinstancer#   	is_activeextendappend)r2   
test_class	test_namedevicedtypeparam_kwargsresult	decorators           r3   get_decoratorszOptimizerInfo.get_decorators   sd     	)I)\2&&	65, MM)"6"67i(	) r5   c                 .    | j                   j                  S r1   )rt   r:   r7   s    r3   namezOptimizerInfo.name   s    ~~&&&r5   )r:   r;   r<   r=   r   r   r?   boolr4   r   propertyr   r@   r5   r3   rP   rP   d   s   6" A2
 ,I3
 !&#(+0  H!%&+&**. $-/gG3G3* sCx+G30 !&c3h1G3> ?G3B !CG3F %)GG3R SG3V  $WG3Z  $[G3^ $(_G3f !c?gG3R
 ' 'r5   rP   c                       e Zd ZdZddZd Zy)optimszGDecorator for specifying a list of optimizers over which to run a test.Nc                 d    t        |      | _        ||| _        y t        j                  g| _        y r1   )listoptim_info_listtorchfloat32dtypes)r2   optim_info_iterabler   s      r3   r4   zoptims.__init__   s*    #$78
 !' 2fr5   c           	   #     K   |t        d      t        j                  | j                  | j                        D ]}  \  }}|j
                  }||d}	 t        j                        fd       }t        j                  |j                  |j                  j                  |j                  |      }	||||	f  y # t        $ r#}
t        d| d|j
                   d       |
d }
~
ww xY ww)NzThe @optims decorator is only intended to be used in a device-specific context; use it with instantiate_device_type_tests() instead of instantiate_parametrized_tests())
optim_infor   c                       | i |S r1   r@   )argsr.   tests     r3   test_wrapperz.optims._parametrize_test.<locals>.test_wrapper   s    000r5   zFailed to instantiate z for module !)rN   	itertoolsproductr   r   r   	functoolswrapspartialr   r:   device_type	Exceptionprint)r2   r   generic_cls
device_clsr   r   r}   r   r   decorator_fnexs    `         r3   _parametrize_testzoptims._parametrize_test   s    3  "+!2!243G3G!U 	J #I +5uEL&1 '1  )00--((MM**  $YlKK-	.  ,YK|JOOCTTUV s+   AC+A&B<9C+<	C(C##C((C+r1   )r:   r;   r<   r=   r4   r   r@   r5   r3   r   r      s    QH$r5   r   c                    t        |       dk(  rt        t        j                  d| |            }t	        t        |i d      t        d      t	        t        ||gi d      t        d	      t	        t        d
|id
|igi d      t        d      t	        t        d t        t        j                  ddg            d      t        d      gS g S )NcpurC   r~   r   zinvalid param typer,   zPparams argument given to the optimizer should be an iterable of Tensors or dictsrK   rL   z.a param group cannot have duplicate parametersz/.*a parameter group with duplicate parameters.*r-   z@duplicate parameters should not occur across param groups eitherz7some parameters appear in more than one parameter groupMbP?)lrzTensor lr must be 1-element)_get_device_typer   r   randnrG   r+   	TypeErrorUserWarning
ValueErrordicttensor)r~   r   sample_params      r3   get_error_inputs_for_all_optimsr      s    5( QvU!KL'-
 %n  (,7I
 'M  %|4x6NO[
 &U  5<<#?@6
 &99%
 %	
N 	r5   c                 l   t        d ddid      t        d dddd      t        d t        j                  d      ddd	      g}t        d i d
      t        d ddid      t        d ddid      t        d ddid      t        d dddd      t        d dddd      gt        |       dk(  r|z   S g z   S )Nrd   Tr,   皙?weight_decayrd   zcapturable with weight decayr   r   rd   Tensor lr with capturabledefaultr   {Gz?non-default lrr   nonzero weight_decayrc   r   rc   maximize, weight_decaygffffff?rS   )rhor   r   cudar+   r   r   r   r~   r   cuda_supported_configss      r3   optim_inputs_func_adadeltar   ;  s    dL$+?lS$'t</	

 	,,u-TB,	
 	d2I>dD$<>NO 5<R	
 	dJ+=JO$'T:)	

 	c B	
 $4F#;v#E	O O LNO Or5   c           
          t        | |      }t        |       dk(  r-|t        t        d t	        dd      d      t
        d      gz  }|S )	Nr   r   g?)r   r   zrho should be between 0 and 1r,   zInvalid rho value: 1.1r   r   r   rG   r+   r   r   r~   r   error_inputss      r3    optim_error_inputs_func_adadeltar   \  sZ    265AL5(4S18
 &4

 
	
 r5   c           	          t        d i d      t        d dddd      t        d dddd	      t        d d
did      t        d ddid      gS )Nr   r,   r   r   )r   r   r   Tr   rc   beta2_decay      znon-default beta2_decayd      ?z non-default clipping threshold d)r+   r   s     r3   optim_inputs_func_adafactorr   m  su    d2I>$'t4'	

 	$'T:	

 	!4(*	

 	:3	
# r5   c                    t        | |      }t        |       dk(  rt        j                  dd| t        j                        }t        j
                  |      |_        |t        t        d t        d      d      t        d	      t        t        d t        d
      d      t        d	      t        t        d t        d      d      t        d	      t        t        |gt               d      t        dt        j                        gz  }|S )Nr         r   )gKHr   )epszepsilon1 should be >= 0r,   r           )r   z	invalid dz#Clipping threshold d should be >= 1皙?)r   zinvalid beta2_decayzbeta2_decay should be <= 0z#does not support complex parametersz-Adafactor does not support complex parameters)rK   rL   rJ   )r   r   r   rand	complex64	rand_likegradrG   r+   r   r   rN   rB   rE   )r~   r   r   complex_params       r3   !optim_error_inputs_func_adafactorr     s   265AL5(

1aeooN"__];N32
 &5  #;$
 &A  C0.
 &8  )?6>
 (K+66	9&
 &	
N r5   c                    t        d i d      t        d ddid      t        d dddd      t        d d	did
      t        d dddd      t        d ddddd      t        d d	t        j                  d      id      gS )Nr   r,   r   r   r   Tr   rc   r   r   )initial_accumulator_valuer   r         ?)r   lr_decayr   r   r   	Tensor lrr+   r   r   r   s     r3   optim_inputs_func_adagradr     s    d2I> 5<R	
 	$'T:	

 	dD#;=MN14cJ,	

 	3D	

 	%,,u-.	
+ r5   c           
          t        | |      }t        |       dk(  r-|t        t        d t	        dd      d      t
        d      gz  }|S )	Nr   r         )r   r   zlr_decay must be bigger than 0r,   zInvalid lr_decay value: -0.5r   r   r   s      r3   optim_error_inputs_func_adagradr     sZ    265AL5(4$79
 &:

 
	
 r5   c           
         t        d ddid      t        d ddddd      t        d t        j                  d      dddd	      g}t        d d
t        j                  d      id      g}t        d i d      t        d d
did      t        d ddid      t        d dddd      t        d dddd      gt        |       dk(  r|ng z   t        |       dk(  r|ng z   }|t        j                  fv r|D ]  }	 d|j
                  d<    |S )Nrd   Tr,   r   )r   amsgradrd   zcapturable, amsgradr   )r   r   rd   z%Tensor lr with capturable and amsgradr   r   r   r   r   r   r   r   rc   )r   r   r   r   mpsr   )r+   r   r   r   float16r.   )r~   r   r   mps_supported_configstotalinputs         r3   optim_inputs_func_adamr     s[   dL$+?lS$'DM&	

 	,,u-$dS8	
 	u||D'9 :	
 $r	B$d|BRS^S$9@V (+>
 (+=	
" &6f%=%G!R#	Q$ %5V$<$E 2%	O 
*    
	&E #&ELL
	& Lr5   c                 4   t        | |      }t        |       dk(  r|t        t        d t	        dd      d      t
        d      t        t        d t	        dd	
      d      t
        d      t        t        d t	        t        j                  d      d      d      t
        d      gz  }t        |       dk(  r_t        j                  d| |      }|t        t        |gdddd      t        d      t        t        |gdddd      t        d      gz  }|S )Nr   r         ?r   r   betasbeta1 should be between 0 and 1r,   &Invalid beta parameter at index 0: 1.0r   r   r   weight_decay should > 0Invalid weight_decay value: -1r   T)r   ra   z7lr as Tensor doesn't work with foreach & not capturablezElr as a Tensor is not supported for capturable=False and foreach=Truer   r@   r   )ra   fusedz/`fused` and `foreach` cannot be `True` together)r   rb   z)`fused` does not support `differentiable`)
r   r   rG   r+   r   r   r   r   emptyrN   )r~   r   r   sample_tensors       r3   optim_error_inputs_func_adamr     s9   265AL5(4z::
 &D  4b92
 &<  5<<#6ER
 &c'
 	
: 6)BvUC)?'+d;J
 (M  )?%)TBD
 (G
 	
( r5   c                    t        d ddid      t        d ddddd      t        d ddddd      t        d dd	ddd
      t        d t        j                  d      dd	ddd      g}t        d i d      t        d ddid      t        d ddid      t        d ddid      t        d dddd      gt        |       dk(  r|z   S g z   S )Nrd   Tr,   rS   r   rc   rd   z"capturable, maximize, weight_decayr   capturable, maximizeFcapturable, weight_decayr   r   r   rc   rd   z#capturable, weight_decay, tensor LRr   r   r   r   r   r   rc   r   r   r   r   r   s      r3   optim_inputs_func_adamaxr  X  s-   dL$+?lS$'TN5	

 	$%4tL'	

 	$'U$O+	

 	ll5) #!"	 7		
#< 	d2I>dD#;=MN 5<R	
 	%	

 	$'T:)	
  $4F#;v#E	!O O  LN!O Or5   c           
          t        | |      }t        |       dk(  r-|t        t        d t	        dd      d      t
        d      gz  }|S )	Nr   r   )r   r   r   zbeta2 should be between 0 and 1r,   z&Invalid beta parameter at index 1: 1.0r   r   r   s      r3   optim_error_inputs_func_adamaxr    sZ    265AL5(4z::
 &D

 
	
 r5   c                     t        | |      S r1   )r   r   s     r3   optim_inputs_func_adamwr    s    !&%00r5   c                     t        | |      S r1   )r   r   s     r3   optim_error_inputs_func_adamwr	    s    '66r5   c                    t        d ddid      t        d dddd      t        d dddd      t        d dddd	d
      t        d t        j                  d      ddddd      g}t        d i d      t        d ddid      t        d ddid      t        d ddid      t        d ddid      t        d ddid      t        d dddd      gt        |       dk(  r|z   S g z   S )Nrd   Tr,   )rc   rd   zmaximize, capturabler   r   weight_decay, capturabler   z"maximize, weight_decay, capturabler   r  z-maximize, weight_decay, capturable, tensor LRr   lambdznon-default lambdr   g{Gz?r   t0d   rc   r   r   r   zmaximize, nonzero weight_decayr   r   r   s      r3   optim_inputs_func_asgdr    sD   dL$+?lS $D9'	

 	$'t<+	

 	$'TN5	

 	ll5) # "	 A		
#: 	d2I>dGS>@STdD$<>NOdD#;TBdJ+=JO 5<R	
 	$'T:1	
 $4F#;v#E	O O LNO Or5   c           
          t        | |      }t        |       dk(  r-|t        t        d t	        dd      d      t
        d      gz  }|S )	Nr   r   r   r   r   r,   z Invalid weight_decay value: -0.5r   r   r   s      r3   optim_error_inputs_func_asgdr    sZ    265AL5(4d;2
 &>

 
	
 r5   c           	          t        d i d      t        d ddid      t        d dt        j                  d      id      t        d dd	id      t        d d
did      gS )Nr   r,   r   r   r   r   r   tolerance_gradư>line_search_fnstrong_wolfer   r   s     r3   optim_inputs_func_lbfgsr    sv    d2I>dD$<>NOu||E': ;+	
 	!14 8?O	
 	$n5	
 r5   c                     t        | |      }|S r1   )r   r   s      r3   optim_error_inputs_func_lbfgsr    s    265ALr5   c                    t        d ddid      t        d ddddd      t        d dddddd	      t        d t        j                  d
      dddddd	      g}t        d i d      t        d dd
id      t        d ddid      t        d ddid      t        d dddd      t        d ddddd      t        d dddd      gt        |       dk(  r|z   S g z   S )Nrd   Tr,   rS   g~jtx?)r   momentum_decayrd   r  )r   r  decoupled_weight_decayrd   z"decoupled_weight_decay, capturabler   )r   r   r  r  rd   r   r   r   r  znon-zero momentum_decayr   r   )r   r  zweight_decay, momentum_decay)r   r  r  r  r   rc   r   r   r   s      r3   optim_inputs_func_nadamr    s`   dL$+?lS$'4tT+	

 	 #"&*."	 6		
 	ll5) #"&*." 6
	
#< 	d2I>dD$<>NO$d+*	

 	  	
 	$'4@/	

 	 #"&*.
 *	
 	$'T:	
;"D $4F#;v#E	E"O "OD LNE"O "Or5   c                     t        | |      }t        |       dk(  rU|t        t        d t	        dd      d      t
        d      t        t        d t	        dd	
      d      t
        d      gz  }|S )Nr   r   r   r   r   r,   r   r   gɿ)r   r  zmomentum_decay should > 0z"Invalid momentum_decay value: -0.2r   r   s      r3   optim_error_inputs_func_nadamr  ;  s    265AL5(4z::
 &D  4=4
 &@
 	
( r5   c                    t        d ddid      t        d dddd      t        d ddddd      t        d t        j                  d	      dddd
d      g}t        d i d      t        d ddid      t        d ddid      t        d ddid      t        d dddd      t        d dddd      gt        |       dk(  r|z   S g z   S )Nrd   Tr,   r   )rd   r   r  )rd   r   r  z0capturable, weight_decay, decoupled_weight_decayr   )r   rd   r   r  z;capturable, weight_decay, decoupled_weight_decay, tensor LRr   r   gMb`?r   r   r  znon-default epsr   r   )r   r  r  r   rc   r   r   r   s      r3   optim_inputs_func_radamr!  V  s/   dL$+?lS" # ,	
 	" #*.
 D	
 	ll5)" #*.	 O		
'> 	d2I>dD$<>NOdE4=?PQ 5<R	
 	$'4H)	

 	$'T:	
" $4F#;v#E	#O O" LN#O Or5   c                     t        | |      }t        |       dk(  rU|t        t        d t	        dd      d      t
        d      t        t        d t	        dd	
      d      t
        d      gz  }|S )Nr   r   r   r   r   r,   r   r   r   r   r   r   r   r   s      r3   optim_error_inputs_func_radamr#    s    265AL5(4z::
 &D  4b92
 &<
 	
( r5   c                    t        d ddid      t        d ddddd      t        d t        j                  d      ddd	      g}t        d i d
      t        d ddid      t        d ddid      t        d ddid      t        d dddd      t        d dddd      t        d ddddd      t        d dddddd      gt        |       dk(  r|z   S g z   S )Nrd   Tr,   r   r   r   r   r   r   r   r   r   r   r   rc   )r   centeredr%  )rc   r   r   )r   r%  momentumr&  )r   r%  r&  rc   z-maximize, centered, weight_decay, w/ momentumr   r   r   s      r3   optim_inputs_func_rmspropr'    sQ   dL$+?lS$'TN'	

 	,,u-TB,	
 	d2I>dD$<>NO 5<R	
 	D 	
 	$'T:	

 	  # *	
 	$'TsK	

 	 #  	 A		
?)R $4F#;v#E	S)O )OR LNS)O )Or5   c           
          t        | |      }t        |       dk(  r-|t        t        d t	        dd      d      t
        d      gz  }|S )	Nr   r   r   r   r&  "momentum should be between 0 and 1r,   zInvalid momentum value: -1.0r   r   r   s      r3   optim_error_inputs_func_rmspropr+    Z    265AL5(4$7=
 &:

 
	
 r5   c           	      *   t        d ddid      t        d t        j                  d      ddd      g}t        d i d      t        d dd	id
      t        d ddid      t        d ddid      t        d ddid      gt        |       dk(  r|z   S g z   S )Nrd   Tr,   r   r   r   r   r   g-C6*?r   etas)r   r   znon-default etas
step_sizes)g>r  znon-default step_sizesrc   r   r   r   s      r3   optim_inputs_func_rpropr0    s    dL$+?lS,,u-TB,	
 	d2I>dD$<>NO 4;M	
 	 +.)	

 	dJ+=JO $4F#;v#E	O O LNO Or5   c           
          t        | |      }t        |       dk(  r-|t        t        d t	        dd      d      t
        d      gz  }|S )	Nr   r   )r   r   )r   r.  z0 < eta1 < 1 < eta2r,   zInvalid eta values: 1.0, 0.5r   r   r   s      r3   optim_error_inputs_func_rpropr2    sZ    265AL5(4j9.
 &:

 
	
 r5   c                 @   t        d i d      t        d ddid      t        d dt        j                  d      id      t        d dd	id
      t        d ddid      t        d dddd      t        d dd	dd      t        d dddd      t        d ddddd      g	S )Nr   r,   r   r   r   r   z	tensor lrr   r   znon-zero weight_decayr&  rS   r   Tr   rc   )r&  	dampeningr4  )r&  r   zweight_decay w/ momentum)r&  nesterovr   r5  r   r   s     r3   optim_inputs_func_sgdr6    s    d2I>dD$<>NOu||E': ;+	
 	 5<S	
 	dJ+<:N$'T:	

 	 ##6	

 	 #S9+	

 	 #sK	
3 r5   c           
          t        | |      }t        |       dk(  r-|t        t        d t	        dd      d      t
        d      gz  }|S )	Nr   r   r   r)  r*  r,   zInvalid momentum value: -0.5r   r   r   s      r3   optim_error_inputs_func_sgdr8  ;  r,  r5   c           	          t        d i d      t        d ddid      t        d dt        j                  d      id      t        d dd	id      gS )
Nr   r,   r   r   r   r   r   rc   Tr   r   s     r3   optim_inputs_func_sparseadamr:  L  s^    d2I>t3C	
 	u||E': ;+	
 	dJ+=JO	 	r5   c                 ,   t        | |      }t        |       dk(  r|t        t        d t	        dd      d      t
        d      t        t        t        j                  d	t        j                  | |
      gi d      t
        d      t        t        dt        j                  d	t        j                  | |
      gigi d      t
        d      t        t        t        j                  dd	| t        j                        gi d      t
        d      gz  }|S )Nr   r   r   r   r   r,   r   r   r   )layoutr~   r   zdense params requiredz+SparseAdam requires dense parameter tensorsr-   z%dense params required in param_groupsr   r   zcomplex not supportedz.SparseAdam does not support complex parameters)r   r   rG   r+   r   r   r   zeros
sparse_coor   r   r   s      r3   "optim_error_inputs_func_sparseadamr?  Y  s+   265AL5(4z::
 &D  e&6&6vU
 0 &I   % %$%+0+;+;+1*/	!"'	 @  &I%(  !JJq!F%//RS0
 &LW4
 4	
j r5   r~   returnc                     t        | t        j                        rt        | j                        } t        | t              sJ | j                  d      d   S )N:r   )rx   r   r~   r?   typesplit)r~   s    r3   r   r     sB    &%,,'V[[!fc"""<<Qr5   r@   c                     t        d D              sJ d       j                         }t         fdj                  D              }g }|D ]  }t	        |j
                        }t        |      dk7  r4|D ]  }	d||	<   	 |j                  t        d||j                               n|j                  |       |D ]>  }	t	        |      }
d|
|	<   |j                  t        d|
|j                   d	|	              @  |S )
a~  
    Return a list of all configs for a given optimizer as a list of OptimizerInputs,
    including configs that have supported global cliquey kwargs (foreach, fused,
    differentiable) based on optim_info.supported_impls.

    The configs (optim_inputs) returned by optim_info.optim_inputs_func(...)
    intentionally do NOT include global cliquey kwargs to give flexibility to tests.
    For example, testing correctness between toggling foreach on and off is now
    trivial. That said, we sometimes want to test for all possible configs on an
    optimizer including all supported flags, so this helper returns all optim inputs.
    c              3   $   K   | ]  }|d v  
 yw))ra   r   rb   Nr@   ).0xs     r3   	<genexpr>zD_get_optim_inputs_including_global_cliquey_kwargs.<locals>.<genexpr>  s      8933s   z?skip must be a subset of ['foreach', 'fused', 'differentiable']c              3      K   | ]@  }|vr:t              j                  v s|d k7  rt              t               v s|dk7  r| B yw)r   ra   N)r   rs   r)   )rG  rH  r~   r   skips     r3   rI  zD_get_optim_inputs_including_global_cliquey_kwargs.<locals>.<genexpr>  sQ      	D=f%)E)EEgV$(N(PPI~ 	
	s   AA	r   FNr,   Tz & )
allrv   tuplerf   r   r.   lenr{   r+   r/   )r~   r   r   rK  optim_inputsrf   all_optim_inputsoptim_inputbase_kwargsflag
new_kwargss   ` ``       r3   1_get_optim_inputs_including_global_cliquey_kwargsrU    s8     =A  IHI  //7L 	++	 	O # {1121$' *$)D!*##d;[EUEUV ##K0 $ 	D!+.J#Jt##
K<L<L;MSQUPV9W	, r5   r`   Tz,Fails fix point assertion on 3.8, see #97811TestOptimRenewedtest_tensor_lr)r   	   )r      )	active_ifzSee #116028)test_set_default_dtype_works_with_foreachzPAccessing grad.real errors, see https://github.com/pytorch/pytorch/issues/117184test_complex_2dg/nB?g-C6
?)rtolatolCompiledOptimizerParityTeststest_correctnessz3This test uses mocks, which dynamo does not support test_defaults_changed_to_foreach)rv   rr   rf   ri   rp   )ra   Fz&See #133268 regarding dtype being Noner   c                 &    | j                  dd      S )Nuse_closureF)getr.   s    r3   r[   r[     s    M5)I r5   )r   rZ  test_can_load_older_state_dict)r   %test_deepcopy_copies_all_public_attrstest_foreach_large_tensortest_foreach_matches_forlooptest_load_nontensor_steptest_mixed_device_dtypetest_param_groups_lrtest_param_groups_weight_decaytest_peak_memory_foreach)test_save_load_equality_with_weights_onlyz(See #116028 regarding copy not supportedtest_state_dict_deterministic test_step_is_noop_for_zero_gradsxpu)rv   rr   rf   rg   rl   rp   )ra   rb   r   )ra   rb   r   rc   rd   )r   r   )r   r   r   c                     t        | dd      S )NwJ?i  rU   rX   rY   s    r3   r[   r[     s    F3h#F r5   c                     t        | d      S )Ng-C6?)	thresholdr^   rY   s    r3   r[   r[     s    -cTB r5   g{Gzt?)r^  r]  test_fused_matches_forloop!test_forloop_goes_right_directionc                     | d    S N
contiguousr@   re  s    r3   r[   r[         VL-A)A r5   )	rv   rr   rf   rg   rs   rh   rk   rq   rp   c                     t        | d      S NrS   rV   r   rY   s    r3   r[   r[     s    s#6 r5   c                     t        | dd      S )N皙?   )start_factortotal_itersr   rY   s    r3   r[   r[     s    #CQG r5   c                     t        | dd      S Nr  r  )factorr  r   rY   s    r3   r[   r[     s    Js3AF r5   c                     t        | d      S r~  r  rY   s    r3   r[   r[         M#S9 r5   c                     t        | d      S r~  r  rY   s    r3   r[   r[     r  r5   c                     t        |       S r1   r^   rY   s    r3   r[   r[     r_   r5   c                     t        | dd      S r  r  rY   s    r3   r[   r[         CC r5   c                     t        | dd      S NrS   r  )powerr  r   rY   s    r3   r[   r[         c!D r5   c                     t        | dd      S rR   rX   rY   s    r3   r[   r[     r\   r5   c                     t        |       S r1   r^   rY   s    r3   r[   r[     r_   r5   )r   r   r   g1E2>gavt>c                 >    t         xr | d   t        j                  k(  S Nr   r(   r   float64re  s    r3   r[   r[   
      )> *57Ou}}4 r5   giUMu>gkNuϵ>TestCudaOptims+test_grad_scaling_autocast_fused_optimizersc                     | d    S rz  r@   re  s    r3   r[   r[   -  r|  r5   zTErrors w/ Global state changed, see https://github.com/pytorch/pytorch/issues/116028)	rv   re   rr   rf   ri   rg   rs   rq   rp   c                     | d    S rz  r@   re  s    r3   r[   r[   W  r|  r5   z2Uses too much memory, even for H100, surprisingly.c                 >    t         xr | d   t        j                  k(  S r  r  re  s    r3   r[   r[     r  r5   c                     | d    S rz  r@   re  s    r3   r[   r[     r|  r5   )rv   rr   rf   rg   rs   ri   rq   rp   giUMu>gh㈵>z7ASGD internally changes the weights even with zero gradg9̗?zDoes not support param groupsz!LBFGS doesn't support multidevice*test_forloop_goes_right_direction_multigpu6test_param_group_with_lrscheduler_goes_right_directionc                 6    t         j                  dk(  xr | d   S )Ndarwinrc  )sysplatformre  s    r3   r[   r[   >  s    )A **=) r5   )rv   rr   rf   rm   rn   ro   rp   c                     | d    S rz  r@   re  s    r3   r[   r[   N  r|  r5   z8Errors, https://github.com/pytorch/pytorch/issues/117150gv!>gw$}>c                     | d    S rz  r@   re  s    r3   r[   r[     r|  r5   gMb@?r   c                     | d    S rz  r@   re  s    r3   r[   r[     r|  r5   c                     t        | dd      S rR   rX   rY   s    r3   r[   r[      s    3"= r5   c                      t        | ddd      S )Nr  r   r  r  
end_factorr  r  rY   s    r3   r[   r[         Hccq r5   c                     t        | dd      S rR   rX   rY   s    r3   r[   r[     r\   r5   c                      t        | ddd      S )Nr  g333333?r  r  r  rY   s    r3   r[   r[     r  r5   c                     t        | dd      S )NGz?rT   rU   rX   rY   s    r3   r[   r[     s    F3dbA r5   c                     t        | d      S )Nr  r  r  rY   s    r3   r[   r[     s    M#T: r5   c                     t        |       S r1   r^   rY   s    r3   r[   r[     r_   r5   c                     t        | dd      S r  r  rY   s    r3   r[   r[     r  r5   c                     t        | dd      S r  r  rY   s    r3   r[   r[     r  r5   c                     t        | dd      S rR   rX   rY   s    r3   r[   r[     r\   r5   c                     t        |       S r1   r^   rY   s    r3   r[   r[     r_   r5   ga2U0*s?)r   rc   r&  r5  r   c                     t        | dd      S )Nrt  i,  rU   rX   rY   s    r3   r[   r[   *  s    7cB r5   gy&1|?)	rv   re   rr   rf   rg   rh   rk   rs   rp   r   g{Gz?z0SparseAdam is not yet supported on the XPU stack)msgz,cannot call to_sparse on p.grad, see #117184z8SparseAdam does not support dense gradients, see #116507 test_state_dict_with_cuda_params)rv   rr   rf   rj   rk   rl   rp   optim_dbc                   *    e Zd ZdZddZd Zd Zd Zy)TensorTrackera0  
    A utility to track tensor clones in a list, with the expectation of popping them later (in
    order) to make fair comparisons between two multi-step computation. The intended use case is
    usually when comparing two supposed equal computations, such as an optimizer step that each
    individually consists of multiple steps, where numerical deviation could multiply.

    The goal is to be able to compare and align numbers at every milestone so as to minimize
    numerical discrepancies, and so when the test fails, it is likely a real problem.
    Nc                 (    |i }|| _         g | _        y r1   )assert_eq_kwargstensors)r2   r  s     r3   r4   zTensorTracker.__init__  s    #! 0r5   c                 r    | j                   j                  |j                         j                                y)z@
        Add a clone().detach()'d version of the tensor
        N)r  r{   clonedetach)r2   r   s     r3   addzTensorTracker.add  s%     	FLLN1134r5   c                    |j                  t        | j                        dd       | j                  j                  d      }|j	                  t        |t              dt        |              |j                  ||fi | j                   t        j                         5  |j                  |       ddd       y# 1 sw Y   yxY w)z
        Pop the first element in the tensor tracker, assert equality between the popped tensor and
        the input tensor, and then set the input tensor to have the same values as the popped tensor
        (with copy_).
        r   zno tensors to popz
type(ref)=N)assertGreaterrN  r  pop
assertTruerx   r
   rC  assertEqualr  r   no_gradcopy_)r2   tensor_to_settestcaserefs       r3   pop_check_setzTensorTracker.pop_check_set  s     	s4<<0!5HIllq!JsF3
S	|_E]CI43H3HI]]_ 	%$	% 	% 	%s    B;;Cc                 2    t        | j                        dk(  S )Nr   )rN  r  r7   s    r3   
all_poppedzTensorTracker.all_popped  s    4<< A%%r5   r1   )r:   r;   r<   r=   r4   r  r  r  r@   r5   r3   r  r    s    5%&r5   r  r1   )NN)r@   )gr   r   r  unittestcopyr   enumr   typingr   r   r   r   r	   r   r
   torch.nnr   torch.optimr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   torch.optim.lr_schedulerr   r   r   r   r   r    *torch.testing._internal.common_device_typer!   r"   2torch.testing._internal.common_methods_invocationsr#   $torch.testing._internal.common_utilsr$   r%   r&   r'   r(   torch.utils._foreach_utilsr)   r+   rB   rG   rP   r   r   r   r   r   r   r   r   r   r   r  r  r  r	  r  r  r  r  r  r  r!  r#  r'  r+  r0  r2  r6  r8  r:  r?  r?   r~   r   rU  version_infor   rK  bfloat16r   r  r   expectedFailurer  __annotations__r  r@   r5   r3   <module>r     s     
    0 0       "  N K  NO O( ' ',Z' Z'z/ /f*NOB"4,^<&5p6r.Ob"17+O\"$
@OF60Of48Ov"O2"D"
9x U3#45  #   %'6	.6v 4 @5!"PQ" **V3Q8H8H68Q	 !-0";
 !f #! !s!'!%( /" !I #2K,
3h 5 A$+FG.""I !"JK"0"	 !"JK"7"	 !"JK"+
 !"JK".
 !"JK"*"	 !"JK")
 !"JK"&"	 !"JK"0"	 !"JK"*
 !"JK";"	 !"LM";
 !"JK"/"	 !"JK"2"	 FG."!	 !"JK"0!	 !"JK"7!	 !"JK"*!	 !"JK"&!	 !"JK"0!	 !"JK";!	 !"JK"/!	 !"JK"2!	CG
O` 3 ?> 
 #q9FB
  "D(As4'@ #,
$ "3A	 !"PQ" **V3Q8H8H68Q	 !-0";
 !f #! !I #23 
MGP 067GHF9
 :2 DDDE@2
$ !=> 
 1 !5==#62O"PQ",5	  "D(As4'@ #,  "s7'C
 !=
1#
J "3A	 !"PQ" **V3Q8H8H68Q	 !j #; !f #! !I #27"
IgP 2 >5"3A	 !"PQ" **V3Q8H8H68Q	 !-0";
 !f #! RS"+
 !I #2=%
,Z 1 => 
 1 !5==#62O"PQ",5	 ! D(As4'@
 #," !s!%!(( !=5&
P "3A	 !"PQ" **V3Q8H8H68Q	 !j #; !f #! !I #27"
kXr 0 <5!"PQ" **V3Q8H8H68Q	 !j #; !f #! !sT'B
 #2 !I #2 M #2K,
3h 1 ="#"' -/O !!'!%* #! =>"&
 =>"0
 AB"<
 =>"H
 !"PQ" **V3Q8H8H68Q	 ((."*	Y3
<z 1 =5"3A	 !"PQ" **V3Q8H8H68Q	 !j #; !f #! !N #* !I #2E)
0b 1 =5!"PQ" **V3Q8H8H68Q	 !j #; !f #! ! sV'D #.	 !I #2?&
-\ 3 ?5"3A	 !"PQ" **V3Q8H8H68Q	 !-0";
 !f #! !sD'A
 #)/	 !I #2G*
1d 1 =5"3A	 !"PQ" **V3Q8H8H68Q	 !-0";
 !f #! !I #23 
'P /=> A B:2
 DDDE@2)
2 !<> 
  !! ! CC	

 !"PQ" **V3Q8H8H68Q	 !j #; !f #! !sE'B
 #)/	 !I #2?&
k\z 6 B#'"D\2." PQ !"PQ"&
 !"PQ" 
 N #0 !"PQ"*
 !"PQ"3
 !"PQ"<
 !"PQ"H
 !"PQ"2
 !"PQ"7e7
@EC!$}
 CL(& (&r5   