sgrI%UddlZddlZddlZddlZddlmZddlmZddlm Z m Z m Z m Z m Z ddlZddlmZddlmZddlmZmZmZmZmZmZmZmZmZmZmZmZmZm Z m!Z!ddl"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z(dd l)m*Z*m+Z+dd l,m-Z-dd l.m/Z/m0Z0m1Z1m2Z2m3Z3dd l4m5Z5Gd dZ6GddeZ7GddZ8GddZ9Gdde/Z:dZ;ddZdZ?ddZ@dZAddZBdZCdd ZDd!ZEdd"ZFd#ZGdd$ZHd%ZIdd&ZJd'ZKdd(ZLd)ZMdd*ZNd+ZOdd,ZPd-ZQdd.ZRd/ZSdd0ZTd1ZUdd2ZVd3ZWd4e eXejfd5eXfd6ZZ dd5e e6fd8Z[e9eekxrejd?kD@e-e1dAde?dMdMdNe-ejdOdHdIdPdQRe-e1dOdkxrejd?kD@e-e1dAdkxrejd?kD@e-e1ddkxrejd?kD@e-e1dAdkxrejd?kD@e-e1ddkxrejd?kD@e-e1ddkxrejd?kD@e-ejdHdId@fe9eeLeMd9d:e-e0dkxrejd?kD@e-e1ddkxrejd?kD@e-e1ddkxrejd?kD@e-e1dAdkxrejd?kD@e-e1dAdkxrejd?kD@e-e1ddrBrD RuntimeErrorr4r@r5r3rGrGNs% SI $66 'r5rGceZdZdZddgfdddddigfdddddd d dd d ed eed fdeed fdedededededededeed ffdZdZ e dZ y ) OptimizerInfoz,Optimizer information to be used in testing.ct|ddSN? gamma step_sizer opts r3zOptimizerInfo.vF3cR@r5ct|Sr1rrYs r3r[zOptimizerInfo.w -c2r5foreachdifferentiable)rarbmaximize capturableFTr@N)scheduler_inputssupported_implsnot_og_supported_flagssupports_sparsehas_capturable_argonly_supports_sparse_gradsmetadata_for_sparsesupports_complexstep_requires_closuresupports_param_groupssupports_multiple_devicesskips decoratorsoptim_error_inputs_funcsupports_fused_on optim_clsrf.rgrhrirjrlrmrnrorsc||_||_||_||_||_||_||_| |_||_| |_ | |_ | |_ | |_ g|r|ng|r|ng|_ ||_||_yr1)rtoptim_inputs_funcrerfrgrhrirkrjrlrmrnrorqrrrs)r2rtrvrerfrgrhrirjrkrlrmrnrorprqrrrss r3r4zOptimizerInfo.__init__gsj#!2 0.&<#."4#6 *D' 0%:"%:")B& &jB e" (?$!2r5cg}|jD]U}t|tr2|j|||||s)|j |jE|j |W|Sr1)rq isinstancer# is_activeextendappend)r2 test_class test_namedevicedtype param_kwargsresult decorators r3get_decoratorszOptimizerInfo.get_decoratorssd )I)\2&& 65,MM)"6"67 i( ) r5c.|jjSr1)rtr:r7s r3namezOptimizerInfo.names~~&&&r5) r:r;r<r=rrr?boolr4rpropertyrr@r5r3rPrPds 6"A2  ,I3 !&#(+0 H!%&+&**. $-/gG3G3*sCx+G30!&c3h1G3>?G3B!CG3F%)GG3RSG3V $WG3Z $[G3^$(_G3f!c?gG3R ''r5rPceZdZdZddZdZy)optimszGDecorator for specifying a list of optimizers over which to run a test.Ncdt||_|||_ytjg|_yr1)listoptim_info_listtorchfloat32dtypes)r2optim_info_iterablers r3r4zoptims.__init__s*#$78 !' 2f  r5c #K| tdtj|j|jD]}\}}|j }||d} t jfd}t j|j|jj|j|} |||| fy#t$r#} td|d|j d| d} ~ wwxYww)NzThe @optims decorator is only intended to be used in a device-specific context; use it with instantiate_device_type_tests() instead of instantiate_parametrized_tests()) optim_inforc|i|Sr1r@)argsr.tests r3 test_wrapperz.optims._parametrize_test..test_wrappers000r5zFailed to instantiate z for module !)rN itertoolsproductrrr functoolswrapspartialrr: device_type Exceptionprint) r2r generic_cls device_clsrrr}rr decorator_fnexs ` r3_parametrize_testzoptims._parametrize_tests  3  "+!2!243G3G!U  J#I+5uEL &1'1 )00--((MM**   $Y lKK- . ,YK|JOOCTTUV  s+AC+A&B<9C+< C(C##C((C+r1)r:r;r<r=r4rr@r5r3rrsQH$r5rct|dk(rttjd||}t t |idt dt t ||gidtd t t d |id |igid td t t dttjd d gdtdgSgS)NcpurCr~rzinvalid param typer,zPparams argument given to the optimizer should be an iterable of Tensors or dictsrKrLz.a param group cannot have duplicate parametersz/.*a parameter group with duplicate parameters.*r-z@duplicate parameters should not occur across param groups eitherz7some parameters appear in more than one parameter groupMbP?)lrzTensor lr must be 1-element) _get_device_typer rrandnrGr+ TypeError UserWarning ValueErrordicttensor)r~r sample_params r3get_error_inputs_for_all_optimsrs5( QvU!KL '- %n  (,7I 'M  %|4x6NO[ &U  5<<#?@6 &9 9% % N r5c ltdddidtdddddtdtjdddd g}tdid tdd d id tdddidtdddidtdddddtdddddgt|dk(r|zSgzS)NrdTr,皙? weight_decayrdzcapturable with weight decayrrrdTensor lr with capturabledefaultr{Gz?non-default lrrnonzero weight_decayrcrrcmaximize, weight_decaygffffff?rS)rhorrcudar+rrrr~rcuda_supported_configss r3optim_inputs_func_adadeltar;sdL$+?lS$'t</ ,,u-TB,   d2I>dD$<>NO 5$'t4' $'T: !4(* :3 # r5c t||}t|dk(rtjdd|tj}tj ||_|ttdtddtd ttdtd d td ttdtddtd tt|gtdtdtjgz }|S)Nrr)gKHr)epszepsilon1 should be >= 0r,r)rz invalid dz#Clipping threshold d should be >= 1皙?)rzinvalid beta2_decayzbeta2_decay should be <= 0z#does not support complex parametersz-Adafactor does not support complex parameters)rKrLrJ)rrrrand complex64 rand_likegradrGr+rrrNrBrE)r~rr complex_params r3!optim_error_inputs_func_adafactorrs265AL5( 1aeooN "__];  N32 &5  #;$ &A  C0. &8  )?6> (K+66 9& & N r5c tdidtdddidtdddddtdd did tdddd d tddd dddtdd tjdidgS)Nrr,rrrTrrcrr)initial_accumulator_valuerr?)rlr_decayrrr Tensor lrr+rrrs r3optim_inputs_func_adagradrsd2I> 5  (+=  "&6f%=%G !R# Q$%5V$<$E 2% O *    &E #&ELL  & Lr5c 4t||}t|dk(r|ttdt dddt dttdt dd d t d ttdt t jd ddt dgz }t|dk(r_t jd||}|tt|gddddtdtt|gddddtdgz }|S)Nrr?rrbetasbeta1 should be between 0 and 1r,&Invalid beta parameter at index 0: 1.0rrrweight_decay should > 0Invalid weight_decay value: -1rT)rraz7lr as Tensor doesn't work with foreach & not capturablezElr as a Tensor is not supported for capturable=False and foreach=Truerr@r)rafusedz/`fused` and `foreach` cannot be `True` together)rrbz)`fused` does not support `differentiable`) rrrGr+rrrremptyrN)r~rr sample_tensors r3optim_error_inputs_func_adamrs9265AL5( 4z:: &D  4b92 &<  5<<#6ER &c '  :6) BvUC  )?'+d;J (M  )?%)TBD (G   ( r5c tdddidtddddddtddddddtddd ddd tdtjd dd dd d g}tdidtdddidtdddidtdddidtdddddgt|dk(r|zSgzS)NrdTr,rSrrcrdz"capturable, maximize, weight_decayrcapturable, maximizeFcapturable, weight_decayrrrrcrdz#capturable, weight_decay, tensor LRrrrrrrrcrrrrrs r3optim_inputs_func_adamaxrXs-dL$+?lS$'TN5 $%4tL' $'U$O+ ll5) #!"  7 #< d2I>dD#;=MN 5dGS>@STdD$<>NOdD#;TBdJ+=JO 5    r5c tdidtdddidtddtjdidtddd idtdd d id gS) Nrr,rrrrrtolerance_gradư>line_search_fn strong_wolferrs r3optim_inputs_func_lbfgsrsvd2I>dD$<>NOu||E': ;+  !14 8?O  $n5  r5ct||}|Sr1)rrs r3optim_error_inputs_func_lbfgsrs265AL r5c tdddidtddddddtddddddd tdtjd ddddd d g}tdid tdd d idtdddidtdddidtdddddtddddddtdddddgt|dk(r|zSgzS)NrdTr,rSg~jtx?)rmomentum_decayrdr )rrdecoupled_weight_decayrdz"decoupled_weight_decay, capturabler)rrrrrdrrrrznon-zero momentum_decayrr)rrzweight_decay, momentum_decay)rrrrrrcrrrs r3optim_inputs_func_nadamrs`dL$+?lS$'4tT+  #"&*."  6  ll5) #"&*." 6 #< d2I>dD$<>NO$d+*    $'4@/  #"&*. *  $'T: ;" D$4F#;v#E E" O"ODLNE" O"Or5c t||}t|dk(rU|ttdt dddt dttdt dd d t d gz }|S) Nrrrrrr,rrgɿ)rrzmomentum_decay should > 0z"Invalid momentum_decay value: -0.2rrs r3optim_error_inputs_func_nadamr;s265AL5( 4z:: &D  4=4 &@   ( r5c tdddidtdddddtddddddtdtjd dddd d g}tdid tdd didtdddidtdddidtdddddtdddddgt|dk(r|zSgzS)NrdTr,r)rdrr)rdrrz0capturable, weight_decay, decoupled_weight_decayr)rrdrrz;capturable, weight_decay, decoupled_weight_decay, tensor LRrrgMb`?rrrznon-default epsrr)rrrrrcrrrs r3optim_inputs_func_radamr!Vs/dL$+?lS" #,   " #*. D  ll5)" #*.  O '> d2I>dD$<>NOdE4=?PQ 5dD$<>NO 5rznon-default step_sizesrcrrrs r3optim_inputs_func_rpropr0sdL$+?lS,,u-TB,  d2I>dD$<>NO 4;M   +.) dJ+=JO $4F#;v#E  O OLN O Or5c t||}t|dk(r-|ttdt dddt dgz }|S) Nrr)rr)rr.z0 < eta1 < 1 < eta2r,zInvalid eta values: 1.0, 0.5rrrs r3optim_error_inputs_func_rpropr2sZ265AL5( 4j9. &:    r5c@tdidtdddidtddtjdidtddd id tdd d id tdd dddtdd d ddtdd d ddtdd dd ddg S)Nrr,rrrrz tensor lrrrznon-zero weight_decayr&rSrTrrc)r& dampeningr4)r&rzweight_decay w/ momentum)r&nesterovrr5rrs r3optim_inputs_func_sgdr6sd2I>dD$<>NOu||E': ;+   5t 3C  u||E': ;+  dJ+=JO  r5c,t||}t|dk(r|ttdt dddt dttt jd t j|| gid t d ttd t jd t j|| gigidt d ttt jdd |t jgidt dgz }|S)Nrrrrrr,rrr)layoutr~rzdense params requiredz+SparseAdam requires dense parameter tensorsr-z%dense params required in param_groupsrrzcomplex not supportedz.SparseAdam does not support complex parameters) rrrGr+rrrzeros sparse_coorrrs r3"optim_error_inputs_func_sparseadamr?Ys+265AL5( 4z:: &D   e&6&6vU 0&I  % % $%+0+;+;+1*/ !"'  @ &I% ( !JJq!F%//RS0 &L W4 4 j r5r~returnct|tjrt|j}t|tsJ|j ddS)N:r)rxrr~r?typesplit)r~s r3rrsB&%,,'V[[! fc "" " << Q r5r@c tdDsJdj}tfdjD}g}|D]}t |j }t |dk7r4|D]} d|| < |jtd||jn|j||D]>} t |} d| | <|jtd| |jd | @|S) a~ Return a list of all configs for a given optimizer as a list of OptimizerInputs, including configs that have supported global cliquey kwargs (foreach, fused, differentiable) based on optim_info.supported_impls. The configs (optim_inputs) returned by optim_info.optim_inputs_func(...) intentionally do NOT include global cliquey kwargs to give flexibility to tests. For example, testing correctness between toggling foreach on and off is now trivial. That said, we sometimes want to test for all possible configs on an optimizer including all supported flags, so this helper returns all optim inputs. c3$K|]}|dv yw))rarrbNr@).0xs r3 zD_get_optim_inputs_including_global_cliquey_kwargs..s89 33sz?skip must be a subset of ['foreach', 'fused', 'differentiable']c3K|]@}|vr:tjvs|dk7rttvs|dk7r|Byw)rraN)rrsr))rGrHr~rskips r3rIzD_get_optim_inputs_including_global_cliquey_kwargs..sQ  D= f %)E)E Eg V $(N(P PI~  sAA rFNr,Tz & ) allrvtuplerfrr.lenr{r+r/) r~rrrK optim_inputsrfall_optim_inputs optim_input base_kwargsflag new_kwargss ` `` r31_get_optim_inputs_including_global_cliquey_kwargsrUs8 =A IHI //7L ++  O# {112  1 $' *$) D! *  # #d;[EUEUV   # #K 0$ D!+.J#Jt   # # Kgavt>c>txr|dtjk(SNrr(rfloat64res r3r[r[ )>*57Ou}}4r5giUMu>gkNuϵ>TestCudaOptims+test_grad_scaling_autocast_fused_optimizersc|d Srzr@res r3r[r[-r|r5zTErrors w/ Global state changed, see https://github.com/pytorch/pytorch/issues/116028) rvrerrrfrirgrsrqrpc|d Srzr@res r3r[r[Wr|r5z2Uses too much memory, even for H100, surprisingly.c>txr|dtjk(Srrres r3r[r[rr5c|d Srzr@res r3r[r[r|r5)rvrrrfrgrsrirqrpgiUMu>gh㈵>z7ASGD internally changes the weights even with zero gradg9̗?zDoes not support param groupsz!LBFGS doesn't support multidevice*test_forloop_goes_right_direction_multigpu6test_param_group_with_lrscheduler_goes_right_directionc6tjdk(xr|dS)Ndarwinrc)sysplatformres r3r[r[>s)A**=)r5)rvrrrfrmrnrorpc|d Srzr@res r3r[r[Nr|r5z8Errors, https://github.com/pytorch/pytorch/issues/117150gv !>gw$}>c|d Srzr@res r3r[r[r|r5gMb@?rc|d Srzr@res r3r[r[r|r5ct|ddSrRrXrYs r3r[r[s3"=r5c t|dddS)Nrrrr end_factorrrrYs r3r[r[Hccqr5ct|ddSrRrXrYs r3r[r[r\r5c t|dddS)Nrg333333?rrrrYs r3r[r[rr5ct|ddS)NGz?rTrUrXrYs r3r[r[ sF3dbAr5ct|dS)NrrrrYs r3r[r[s M#T:r5ct|Sr1r^rYs r3r[r[r_r5ct|ddSrrrYs r3r[r[rr5ct|ddSrrrYs r3r[r[rr5ct|ddSrRrXrYs r3r[r[r\r5ct|Sr1r^rYs r3r[r[r_r5ga2U0*s?)rrcr&r5rct|ddS)Nrti,rUrXrYs r3r[r[*s7cBr5gy&1|?) rvrerrrfrgrhrkrsrprg{Gz?z0SparseAdam is not yet supported on the XPU stack)msgz,cannot call to_sparse on p.grad, see #117184z8SparseAdam does not support dense gradients, see #116507 test_state_dict_with_cuda_params)rvrrrfrjrkrlrpoptim_dbc*eZdZdZddZdZdZdZy) TensorTrackera0 A utility to track tensor clones in a list, with the expectation of popping them later (in order) to make fair comparisons between two multi-step computation. The intended use case is usually when comparing two supposed equal computations, such as an optimizer step that each individually consists of multiple steps, where numerical deviation could multiply. The goal is to be able to compare and align numbers at every milestone so as to minimize numerical discrepancies, and so when the test fails, it is likely a real problem. Nc(|i}||_g|_yr1)assert_eq_kwargstensors)r2rs r3r4zTensorTracker.__init__s  #!  0 r5cr|jj|jjy)z@ Add a clone().detach()'d version of the tensor N)rr{clonedetach)r2rs r3addzTensorTracker.adds% FLLN1134r5c|jt|jdd|jjd}|j t |t dt||j||fi|jtj5|j|dddy#1swYyxYw)z Pop the first element in the tensor tracker, assert equality between the popped tensor and the input tensor, and then set the input tensor to have the same values as the popped tensor (with copy_). rzno tensors to popz type(ref)=N) assertGreaterrNrpop assertTruerxr rC assertEqualrrno_gradcopy_)r2 tensor_to_settestcaserefs r3 pop_check_setzTensorTracker.pop_check_sets s4<<0!5HIllq!JsF3 S |_E]CI43H3HI ]]_ %    $ % % %s B;;Cc2t|jdk(S)Nr)rNrr7s r3 all_poppedzTensorTracker.all_poppeds4<< A%%r5r1)r:r;r<r=r4rrrr@r5r3rrs 5 %&r5rr1)NN)r@)grrrunittestcopyrenumrtypingrrrrr rr torch.nnr torch.optimr r rrrrrrrrrrrrrtorch.optim.lr_schedulerrrrrrr *torch.testing._internal.common_device_typer!r"2torch.testing._internal.common_methods_invocationsr#$torch.testing._internal.common_utilsr$r%r&r'r(torch.utils._foreach_utilsr)r+rBrGrPrrrrrrrrrrrrrr rrrrrrr!r#r'r+r0r2r6r8r:r?r?r~rrU version_inforrKbfloat16rrrexpectedFailurer__annotations__rr@r5r3rs 00 "NKNOO('',Z'Z'z/ /f*NOB"4,^<&5p6r.Ob"17+O\"$ @OF60Of48Ov"O2"D" 9x U3 #45 # %'6 .6v4 @5 !"PQ" **V3Q8H8H68Q   !-0";  !f#!   ! s!'!%(/"  !I#2  K, 3h5 A$+  FG.""I   !"JK"0"   !"JK"7"   !"JK"+  !"JK".  !"JK"*"   !"JK")  !"JK"&"   !"JK"0"   !"JK"*  !"JK";"   !"LM";  !"JK"/"   !"JK"2"    FG."!   !"JK"0!   !"JK"7!   !"JK"*!   !"JK"&!   !"JK"0!   !"JK";!   !"JK"/!   !"JK"2!  CG O`3 ?> #q 9FB    "D(A s4'@ #,  $ "3A   !"PQ" **V3Q8H8H68Q   !-0";  !f#!   !I#2  3  MGP 0 6 7 G HF9  :2 D D D E@2  $!=> 1 !5==#62O"PQ",5    "D(A s4'@ #,  " s7'C != 1# J "3A   !"PQ" **V3Q8H8H68Q   !j#;   !f#!   !I#2  7" IgP2 >5 "3A   !"PQ" **V3Q8H8H68Q   !-0";  !f#!    RS"+  !I#2  =% ,Z 1 => 1 !5==#62O"PQ",5   !D(A s4'@ #, " ! s!%!((!= 5& P "3A   !"PQ" **V3Q8H8H68Q   !j#;   !f#!   !I#2  7" kXr 0 <5 !"PQ" **V3Q8H8H68Q   !j#;   !f#!   ! sT'B #2  !I#2    M#2  K, 3h 1 ="#"' -/O  !!'!%*#!   =>"&   =>"0   AB"<   =>"H  !"PQ" **V3Q8H8H68Q   ((."*  Y3 <z 1 =5 "3A   !"PQ" **V3Q8H8H68Q   !j#;   !f#!   !N#*   !I#2  E) 0b 1 =5 !"PQ" **V3Q8H8H68Q   !j#;   !f#!   ! sV'D #.  !I#2  ?& -\3 ?5 "3A   !"PQ" **V3Q8H8H68Q   !-0";  !f#!   ! sD'A #)/  !I#2  G* 1d 1 =5 "3A   !"PQ" **V3Q8H8H68Q   !-0";  !f#!   !I#2  3  'P / = >  A B:2  D D D E@2 ) 2!<> !! !  C C   !"PQ" **V3Q8H8H68Q   !j#;   !f#!   ! sE'B #)/  !I#2  ?& k\z6 B#'"D\2. "  PQ  !"PQ"&  !"PQ"    N#0   !"PQ"*  !"PQ"3  !"PQ"<  !"PQ"H  !"PQ"2  !"PQ"7 e7 @EC!$} CL(&(&r5