sgVUddlmZmZmZmZddlZddlmcmZ ddlm Z gZ ee e d<ejjGddZy))DictListOptionalTupleN)Tensor__all__ceZdZ ddeededeeefdededededed ed efd Zd ed e efdZ dee efdZ y)_FunctionalAdamparamslrbetaseps weight_decayamsgradmaximizeforeachfused_allow_empty_param_listc Nd|kstd|d|kstd|d|dcxkrdksntd|dd|dcxkrdksntd|dd|kstd ||||d|d|d |_||_||_||_| |_t jjtt jttt jffi|_ t|dk(r | s td d |i|_y) NzInvalid learning rate: zInvalid epsilon value: rg?z#Invalid beta parameter at index 0: z#Invalid beta parameter at index 1: zInvalid weight_decay value: )r rbeta1beta2rz%optimizer got an empty parameter listr ) ValueErrordefaultsrrrrtorchjitannotaterrstrstatelen param_group) selfr r r rrrrrrrs Z/var/www/html/venv/lib/python3.12/site-packages/torch/distributed/optim/functional_adam.py__init__z_FunctionalAdam.__init__sBby6rd;< <cz6se<= =eAh$$B58*MN NeAh$$B58*MN Nl";L>JK K1X1X(        YY''U\\4U\\@Q;R-R(SUWX v;! $;DE E%f-paramgradcfg}g}g}g}g}g}tj|} |"|j||j|||jvri|j|<|j|} tjd| d<tj |tj | d<tj |tj | d<|jr(tj |tj | d<|j|} |j| d|j| d|jr|j| d|j| dtj5tj|||||||j| |j|jd|jd |jd |jd |jd |j|jdd dddy#1swYyxYw)zo Similar to step, but operates on a single parameter and optionally a gradient tensor. Nrstep memory_formatexp_avg exp_avg_sqmax_exp_avg_sqrrr rr r has_complexrrrr rrrr grad_scale found_inf)r is_complexappendr tensor zeros_likepreserve_formatrno_gradFadamrrrr) r#r'r(params_with_gradgradsexp_avgs exp_avg_sqsmax_exp_avg_sqs state_stepsr1r s r$ step_paramz_FunctionalAdam.step_paramCs  $& &&u-    # #E * LL   " "DJJu JJu%E!LL-E&M$//U%:%: E) #("2"2U%:%:#E, ||*/*:*:)>)>+&' 5!i()5./ <<  " "5)9#: ;5=) ]]_  FF  'mmG,mmG,==&!]]>:MM%( jj%    s BH''H0 gradientscX|jd}g}g}g}g}g}g}d} t|t|k7r*tddt|dzdt|zt|jd|D]v\} } |  | t j | z} |j | |j | | |jvri|j| <|j| } t jd| d<t j| tj | d <t j| tj | d <|jr(t j| tj | d <|j| } |j | d |j | d |jr|j | d |j | dyt j5tj|||||||j| |j|j d |j d|j d|j d|j d|j"|j$dddddy#1swYyxYw)Nr FzEthe gradients passed in does not equal to the size of the parameters!zParams length: z. zGradients length: rr*r+r-r.r/rrr rrr0)r"r!rziprr4r5r r6r7r8rr9r:r;rrrr) r#rCr r<r=r>r?r@rAr1r'gradientr s r$r*z_FunctionalAdam.steps!!(+ $&  v;#i. (W#CK=34&s9~&678  #4#3#3H#=yI 2OE8#u//66  ''. X& *(*DJJu% JJu-E$)LL$5E&M','7'7U-B-B(E)$+0*:*:U-B-B+E,'||272B2B!1F1F3./ 5)i 01""5#67<<#**51A+BC""5=1A 2D]]_  FF  'mmG,mmG,==&!]]>:MM%( jj%    s BJ  J)N) gMbP?)g?g+?g:0yE>rFFFFF) __name__ __module__ __qualname__rrfloatrboolr%rrBr*r&r$r r s %1!(-*.V *. *.UE\" *.  *.  *.*.*.*.*."&*.X::hv.>:xGd8F#34Gr&r )typingrrrrrtorch.optim._functionaloptim _functionalr:rrr__annotations__rscriptr rLr&r$rSsM.. ##cpppr&