
    sg:                        d dl Z d dlZd dlZd dlZd dlmZmZ d dlmZm	Z	m
Z
 d dlZddlmZmZ ddlmZmZmZmZ ddlmZmZ ddlmZ dd	lmZmZmZmZmZm Z m!Z! dd
l"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z,m-Z- ddl.m/Z/m0Z0m1Z1 ddl2m3Z3 ddl4m5Z5 ddl6m7Z7 ddl8m9Z9 e
rd dl:m;Z; ddZ<ede=dej|                  j~                  fd       Z@d ZA G d de1      ZB G d de9      ZC G d deC      ZD G d deC      ZEy)     N)contextmanagernullcontext)DictListTYPE_CHECKING   )trace_rules	variables)raise_observed_exceptionunimplementedUnspecializeRestartAnalysisUnsupported)GuardBuilderinstall_guard)GenerationTracker)
AttrSourceConstDictKeySourceFSDPNNModuleSourceGetItemSourceNNModuleSource"UnspecializedBuiltinNNModuleSourceUnspecializedNNModuleSource)get_custom_getattrget_fake_valueis_lazy_moduleis_namedtupleis_safe_constantistensoristypennmodule_has_hooksobject_has_getattributeproxy_args_kwargsset_example_value   )MutableLocaltypestrVariableTracker)invoke_and_store_as_constant)LazyVariableTracker)SliceVariable)UserDefinedObjectVariable)InstructionTranslatorc                     t        |d      rg fdt        ||      \  }}|D cg c]
  } |       }}|j                         D 	ci c]  \  }}	| |	       }
}}	|j                  |||
       yyc c}w c c}	}w )a~  
    Fairly coupled helper used by NNModuleVariable and UnspecializedNNModuleVariable.

    Used to cause lazy module to be initialized (and delete its init hook) before tracing. Especially
    useful now that 'allowed' modules graph-break on hooks, calling this first ensures there is no hook
    by the time we trace __call__ and thus no graph-break for lazy allowed modules.
    _initialize_hookc                    t        |       r t        |       fd| D         S t        | t              r*| j	                         D ci c]  \  }}| |       c}}S t        | t
        t        t        f      r t        |       fd| D              S t        | t        j                  j                        rt        | j                        S | S c c}}w )Nc              3   .   K   | ]  } |        y wN .0elemconvert_to_fakes     T/var/www/html/venv/lib/python3.12/site-packages/torch/_dynamo/variables/nn_module.py	<genexpr>zBinitialize_lazy_module.<locals>.convert_to_fake.<locals>.<genexpr>B   s      E4!6 E   c              3   .   K   | ]  } |        y wr1   r2   r3   s     r7   r8   zBinitialize_lazy_module.<locals>.convert_to_fake.<locals>.<genexpr>F   s     Ct4Cr9   )r   type
isinstancedictitemslisttuplesettorchfxProxyr   node)xkvr6   txs      r7   r6   z/initialize_lazy_module.<locals>.convert_to_fake@   s    QtAw E1 EFFAt$:;'')D$!Q?1--DDAeS12tAwCCCCAuxx~~.%affb11 Es   CN)hasattrr"   r>   _infer_parameters)rI   modargskwargs
proxy_argsproxy_kwargsarg	fake_argsrG   rH   fake_kwargsr6   s   `          @r7   initialize_lazy_modulerT   6   s     s&'
	 $5T6#B 
L5?@c_S)@	@9E9K9K9MNAq/!,,NNc9k:# ( ANs   A7A<
module_keyrL   c              #      K   |j                         }	 ||j                  f|j                  | <   d  |j                  | = y # |j                  | = w xY wwr1   )name	__class__nn_module_stack)rU   sourcerI   rL   fully_qualified_names        r7   record_nn_module_stackr\   R   sO     !;;=+*>)N:&z*Bz*s   AA AAAc                    | rd|j                   v rt        |j                   d         rk|j                   d   }t        | d      }t        |      t        j
                  u rt        |d      }t        |j                  t        j                               y t        | j                  t        j                  t        j                  d                   y y )Nforward__func__)attr)__dict__callabler   r;   types
MethodTyper   
make_guardr   CLOSURE_MATCH	functoolspartialNOT_PRESENT_IN_GENERIC_DICT)rZ   rL   fwdforward_sources       r7   &guard_to_detect_forward_monkeypatchingrl   \   s     $#,,y2I)J,,y)C'	:NCyE,,,!+NJ!G.33L4N4NOP !!%%$@@y     c                       e Zd Zddddhej                  Zdededej                  j                  ddf fdZ
d Zd	 Zd
 Z	 	 ddZd ZdddeddfdZd Zd ZddZd ZddZ	 	 	 	 	 	 ddZ	 d	 	 	 	 	 d fdZ xZS )NNModuleVariablemodule_typerU   modulenn_module_stack_sourcereturnNc                     t        |   di | || _        || _        || _        | j
                  sJ | j
                  | _        y )Nr2   )super__init__rp   rU   rq   rZ   rr   )selfrp   rU   rq   rN   rX   s        r7   rv   zNNModuleVariable.__init__   sC     	"6"&${{{&*kk#rm   c                 6    | j                   xs | j                  S r1   rr   rZ   rw   s    r7   get_nn_module_stack_sourcez+NNModuleVariable.get_nn_module_stack_source       **9dkk9rm   c                     || _         y r1   rr   rw   rZ   s     r7   set_nn_module_stack_sourcez+NNModuleVariable.set_nn_module_stack_source   
    &,#rm   c                     | j                   S r1   )rp   rz   s    r7   python_typezNNModuleVariable.python_type   s    rm   rI   r,   c                      y r1   r2   )rw   rI   rZ   submod	key_extraoptionss         r7   _wrap_submodulez NNModuleVariable._wrap_submodule   s     	rm   c                 z   |j                   j                  | j                        }t        |t        j
                  j                        rg }|j                         D ]{  \  }}t        j                  j                  |      }|j                   j                  || j                  |t        t        | j                  |                   |j                  |       } |S t        |t        j
                  j                   t        j
                  j"                  t        j
                  j$                  f      sJ t'        |             | j                  sJ g }t)        |      D ]Z  \  }}|j                  |j                   j                  || j                  |t        t        | j                  |                         \ |S NrZ   )outputget_submodulerU   r<   rB   nn
ModuleDictr>   r
   ConstantVariablecreateregister_attr_or_moduler   r   rZ   append
ModuleListParameterList
Sequentialr&   	enumerate)rw   rI   baseresultrW   r   name_varidxs           r7   unpack_var_sequencez$NNModuleVariable.unpack_var_sequence   sj   yy&&t7dEHH//0F $

 (f$55<<TB		11OO)-T*JK	 2  h'( M588&&(>(>@S@ST
 	4=	 
 {{{$T? 	KCMM		11OO)-S*IJ	 2 	 rm   rW   r'   c                 ,   |j                   j                  | j                        }t        ||      }t	        t        t        | j                  |            j                  t        j                               t        j                  j                  |      S r1   )r   r   rU   rJ   r   r   r   rZ   re   r   HASATTRr
   r   r   )rw   rI   rW   rL   r   s        r7   call_hasattrzNNModuleVariable.call_hasattr   sn    ii%%doo6d#:dkk489DD$$	

 ))0088rm   c                 f    |j                   j                  | j                        }t        |dd      S )NtrainingF)r   r   rU   getattrrw   rI   rL   s      r7   is_trainingzNNModuleVariable.is_training   s)    ii%%doo6sJ..rm   c                     |j                   j                  | j                        }t        j                  |       |j
                  j                  dk7  rt        j                  t        |             t        )zIRestart analysis treating this module as an UnspecializedNNModuleVariablerv   )
r   r   rU   r   tagf_codeco_namemark_class_dynamicr;   r   r   s      r7   convert_to_unspecializedz)NNModuleVariable.convert_to_unspecialized   sT    ii%%doo6c" 99
*00c;))rm   c                 ~   |j                   j                  | j                        }t        |      rt	        d       |j                   j
                  j                  | |      rC|j                   j
                  j                  | |d      }t        |t        j                         S t        j                  |d      }||v S )Nz-NNModuleVariable with custom __getattribute__T)
deleted_okra   )r   r   rU   r!   r   side_effectshas_pending_mutation_of_attr	load_attrr<   r
   DeletedVariableobject__getattribute__)rw   rI   keyr   mutated_attr	base_dicts         r7   has_key_in_generic_dictz(NNModuleVariable.has_key_in_generic_dict   s    yy&&t7"4(IJ99!!>>tSI9911;;D#RV;WL!,	0I0IJJJ++D*=	irm   c                 $   t        |      rt        d       t        |d      }|yt        |t        j
                        st        d       t        j                  || fi |j                  |t        j                  j                  |      gi       S )zDCheck for a __getattr__ and handle it specially if it is implementedz6torch.nn.Module with a custom __getattribute__ definedT)ignore_nn_module_getattrNz6torch.nn.Module with a non-function custom __getattr__)r!   r   r   r<   rc   FunctionTyper
   UserMethodVariablecall_functionr   r   )rw   r   rI   rW   r   
getattr_fns         r7   _custom_getattr_fallbackz)NNModuleVariable._custom_getattr_fallback   s    "4(RS'tL
*e&8&89RS++JHHVV++224892
 	
rm   c           
      z   ddl m} | j                  rt        | j                  |      }nd }|j                  j                  | j                        }t        j                  |d      }d}t               }t        j                  |j                        D ]+  }	|j                  |	j                  j                                - | j                  st!        d       |dk(  rt#        j$                  | ||      S ||v r||   }
nYd|v r||d   v r||vr	|d   |   }
nAd|v r||d   v r	|d   |   }
n-d	|v r||d	   v r	|d	   |   }
n	 t        j&                  ||      }
d
}|dk(  rt-        | j                  |       |dk(  r#|s!t#        j.                  |j                  |      S |rY  ||t1        |            |
      }t3        |t4        t6        f      r)|j9                  t        | j;                         |             |S t=        |
t>              rk| j                  r,t        t        | j                  d      |      }t        |d      }t#        j@                  |
jB                  |      jE                  || gi       S t=        |
tF              r>t#        jH                  |
jJ                  t#        jL                  tO        |            |      S t=        |
tP              r&t#        j@                  |
jS                  |      |      S t=        |
tT        jV                        rt#        jH                  |
| |      S tY        |
      st[        |
      r  ||t1        |            |
      S t!        d| dt]        |       dt]        |
              t#        j$                  | ||      S # t(        $ r | j+                  |||d|i      }||cY S  w xY w)Nr$   )VariableBuilderra   TzGETATTR with no sourcer   _modules_parameters_buffersFrZ   )r   rI   rW   r   r^   rX   fgetzclass property z -  )/builderr   rZ   r   r   r   rU   r   r   rA   inspectgetmrorX   updatera   keysr   r
   GetAttrVariablegetattr_staticAttributeErrorr   rl   UserDefinedClassVariabler   r<   ro   UnspecializedNNModuleVariabler   r{   r   propertyUserFunctionVariabler   r   classmethodr   r_   r+   r;   staticmethod__get__rc   r   r   r   r&   )rw   rI   rW   r   rZ   r   r   object_memberall_class_attribute_namesrF   subobjr   outs                r7   var_getattrzNNModuleVariable.var_getattr   s   ,;;T2FFyy&&t7++D*=	$'E!/ 	@A%,,QZZ__->?	@ {{23:,,T4GG9t_F)#	*--55z*40Fi'DIm4L,L}-d3F9$:1F)Fz*40F //d; % 924;;E;}55dnnVTT=/"nV&<=fEC# 02OPQ ..t>>@$G J fh';;'
4;;(LdSF'7F 55KK!  -dHb12 , 33OO77T
C! 
 - 55NN4(   2 23 33FDPP!&)Xf-=Br>&+AB6JJ%dV3wt}oQwv>OP ((tFCCw " 66"4(F9K 7  %!Ms   ?N $N:8N:c                    |j                   j                  | j                        }t        | j                  | j	                         ||      5  t        |      }t        |t        j                  j                        r|j                  j                  t        j                  j                  j                  u rt        |      r| j                  |       |rJ d       |rJ |\  }|j                  j                         D ]m  \  }}|j!                  |j                   j#                  || j                  |t%        t'        | j(                  |                  |gi        |j+                         }o |cd d d        S |r+|j,                  |j,                  | _        t1        ||||       |j                   j3                         r|j4                  j7                  d      rw|j4                  dk7  rht        |dd      r| j                  |       ddlm}	  |	| |j                   j<                  d	| j                  gt?        ||       
      cd d d        S | j(                  sJ d       t        |t        j@                  jB                        r#|j                  }
t'        | j(                  d      }n"|jD                  }
t'        | j(                  d      }tG        |
tH        jJ                        r|
jL                  }
t'        |d      }| g|z   }ntG        |
tH        jN                        sJ |jQ                  tS        jT                  |
|      ||      cd d d        S # 1 sw Y   y xY w)Nz3Expected lazy sequential isn't a valid combination?r   z	torch.nn.z	torch.ao.ztorch.nn.utils.parametrizeT)check_forward_hookscheck_backward_hooksr$   wrap_fx_proxycall_modulerI   proxyzsMust provide a valid source in order to inline, since inlined function may have default args which must be guarded.r^   
_call_implr_   )+r   r   rU   r\   r{   r   r<   rB   r   r   rX   r^   r    r   r   r>   r   r   r   r   rZ   popcls_to_becomerp   rT   is_root_tracer
__module__
startswithr   r   create_proxyr"   rC   GraphModuler   r   rc   rd   r_   r   inline_user_function_returnr
   r   )rw   rI   rM   rN   rL   is_lazyrQ   
child_namer   r   fn	fn_sources               r7   r   zNNModuleVariable.call_functionU  s    ii%%doo6#OOT<<>C
 a	 %S)G3 3 34MM))UXX-@-@-H-HH%c* 11"5  IHI!!z*-,,*<*<*> #&J$$		99" OO&#1*T[[*2U#V	 :  	 &&(C# Ca	 a	F $$0'*'8'8D$
 'r3f= 		((*NN--.HINN&BB%T 11"52$0"))00% +48Ca	 a	T {{ Z{ c588#7#78 B *4;;	 BIB *4;; EI"e../B *9j AI 6D=D!"e&8&899955222iH{a	 a	 a	s   D"M8CMC8MMc                     ddl mmm  j                  j
                  j                         fd}dv r j                        S dk(  r8t         j                   j                               5   |      cd d d        S dk(  rMt        j                  t        j                  j                  j                              rj!                  d      S dk(  rd   j#                         sJ t%        d	         sJ d	   j&                  d   j(                     }t%        |t*              r|S |j                  j
                  j                        }j
                  j-                  |t/        t1         j2                              
      S |r7t5              }	j                  j6                   d dt9        |	      S fdfd}
 fd} fd}d dk(  rj
                  j:                  j=                  t?         j2                  d      jA                                srJ g }jC                         D ]  \  }|jE                   ||               |tG                     S dk(  rj
                  j:                  j=                  t?         j2                  d      jA                                g } jH                  d0i  |
dd      D ]  \  }|jE                   ||               |tG                     S dk(  rj
                  j:                  j=                  t?         j2                  d      jA                                g } jJ                  d0i  |
ddd      D ]  \  }|jE                   ||               |tG                     S dk(  rj
                  j:                  j=                  t?         j2                  d      jA                                g } jL                  d0i  |
ddd      D ]  \  }|jE                   ||               |tG                     S dk(  rcj
                  j:                  j=                  t?         j2                  d      jA                                srJ  |jC                               S dk(  r]j
                  j:                  j=                  t?         j2                  d      jA                                 |jM                               S d k(  rej
                  j:                  j=                  t?         j2                  d      jA                                 | jH                  d0i  |
d            S d!k(  rej
                  j:                  j=                  t?         j2                  d      jA                                 | jJ                  d0i  |
d            S d"k(  rOsrJ g }jO                         D ]"  |jE                  j!                               $  |tG                     S d#k(  rsrJ  |j'                               S d$k(  rJsrJ g }j'                         D ]  \  }|jE                   ||               |tG                     S d%k(  r srJ j!                  tQ                    S d&k(  rt%        tR        jT                  jV                  tR        jT                  jX                  f      rCrAd	   j#                         r.j!                  d	   j[                         j\                  v       S d'k(  rωstQ              dk(  sJ tR        jT                  jV                  j^                  tR        jT                  j`                  j^                  tR        jT                  jX                  j^                  tR        jT                  jb                  j^                  tR        jT                  jd                  j^                  f}tg              j^                  |vrt%        d	   th        j                        sJ tk        d	                d	   j[                         t%        tl        tn        f      sJ t5              jp                  }	t%        |	tr        jt                        sJ t?        t?         j2                        d(      }jw                  ti        jx                  |	|
       gt{              z         S  j2                  sJ t%        d	   t|              r:j
                  j~                  rg }t{        t        tQ                          d	   j[                            }t        d	   j[                                  D ]V  \  }}||   t/        t1         j2                              }|jE                  j
                  j-                  ||
             X d	   j[                            }j
                  j-                  |  d)t/        t1         j2                  d	   j[                                     
      }|S  j                         dd*lCmD} t%        d	   |      rd	   j                  j
                        n8d	   j#                         rd	   j[                         nt        d+d	              }j
                  j-                  | j                  t/        t1         j2                              
      S d,k(  szt%        tR        jT                  j                  j                  j                        rd-k(  s=t%        tR        jT                  j                  j                  j                        rgd.k(  rbt5              jp                  }	t?        t?         j2                        d(      }jw                  ti        jx                  |	|
       gz         S j                  j                  v r^t        j                  j                           r<t        d/ t        j                  j                               D              r |      S t         I        S # 1 sw Y   )xY w)1Nr$   )r   ListIteratorVariableTupleVariablec           	          	j                   j                  dj                  di       }t        |j                         t              \  }}ddlm}  |		j                   j                  d| |g||            S )Nget_attrr2   r$   r   call_method)rM   rN   r   )r   r   rU   r#   rE   r"   r   r   )
rW   	mod_proxyrO   rP   r   rM   rN   rq   rw   rI   s
        r7   generic_call_method_helperz@NNModuleVariable.call_method.<locals>.generic_call_method_helper  s     		..	I innf5'8v'F$J. ii,,!#1j1'	 -  rm   r   _wrapped_call_implr^   _check_input_dimT_get_item_by_idxr   r   __resultc                      t        d t        j                   j                               D              st	        d        y y )Nc              3   <   K   | ]  }|j                           y wr1   )is_python_constantr4   rF   s     r7   r8   zUNNModuleVariable.call_method.<locals>.assert_all_args_kwargs_const.<locals>.<genexpr>  s      +,$$&s   znon-const NNModule method )all	itertoolschainvaluesr   )rM   rN   rW   s   r7   assert_all_args_kwargs_constzBNNModuleVariable.call_method.<locals>.assert_all_args_kwargs_const  s@     09fmmo0V   :4&ABrm   c                              t        	
      } t        j                  |      j                  D cg c]  }|j	                          c}i j                         D ci c]  \  }}||j	                          c}}}|j                          |j                  }| D ci c]  }|||   
 c}S c c}w c c}}w c c}w r1   )r   r   	signaturebindas_python_constantr>   apply_defaults	arguments)namesr   rF   rG   rH   
bound_argsrM   r  rN   rq   rW   s         r7   
get_kwargsz0NNModuleVariable.call_method.<locals>.get_kwargs  s    (*&B3**2.33378a1'')89?HA1a**,,HJ %%'#--J.34Az!}$44 9H 5s   B6%B;&Cc                     g }| D ]M  \  }}|j                  j                  j                  ||t         j                  |                         O  |t                     S )Nr   mutable_local)r   r   r   r   rZ   r%   )	r>   r   rW   r   r   
gen_sourcer   rw   rI   s	       r7   wrap_valuesz1NNModuleVariable.call_method.<locals>.wrap_values  sk    F % fII55-jd.KL	 6  (lnMMrm   c                      j                  |       j                  j                  || t         j                  |                   g      S r   )r   r   r   r   rZ   )rW   objr   r   r  r   rw   rI   s     r7   named_embedz1NNModuleVariable.call_method.<locals>.named_embed+  sV     $++D1II55-jd.KL	 6 
 
rm   c                     |j                  d      }|d   dk(  r| S t        |      dkD  r,|j                  d      }t        | |      } t        |      dkD  r,| S )N.r    )splitlenr   r   )rZ   rW   
name_splitrF   s       r7   r  z0NNModuleVariable.call_method.<locals>.gen_source8  s[    CJ!}"j/A%NN1%#FA. j/A% Mrm   named_childrenr   r  named_parametersr   prefixrecursenamed_buffersr   remove_duplicatenamed_modulesmemochildrenmodules
parametersbuffersr   r  r>   __len____contains____getitem__r_   z.__getitem__(slice))SymNodeVariablez%getitem on NNModuleVariable with key _get_abs_string_index_conv_forward_output_paddingc              3   P   K   | ]  }t        |t        j                           y wr1   )r<   r
   TensorVariabler   s     r7   r8   z/NNModuleVariable.call_method.<locals>.<genexpr>  s%       1i667s   $&r2   )Sr  r   r   r   rU   r   r   r   r\   r{   r	   is_torch_inline_allowedr   getfilerX   r   r   r   r<   r>   valuer   r   r   r   rZ   r   __name__r(   guard_on_key_orderaddr   rW   r  r   r%   r  r!  r#  r   r  rB   r   r   ParameterDictr	  r   r+  r   r   r   r;   r
   r&   strintr_   rc   r   r   r   r?   r*   exportranger   r   tensorr,  evaluate_exprr   r&  conv_ConvNd_ConvTransposeNdra   rb   r  r  r  r  ru   r   ) rw   rI   rW   rM   rN   constantr   mod_varr   r   r  r  r  r   parambufferbuiltin_supportedsrcr   r   
new_modulenew_module_variabler,  r   r   r   r   r  r  r   rq   rX   s    `````                   @@@@@@@r7   r   zNNModuleVariable.call_method  sa    	LKoo((-	 	2 77 %%b$77Y (!@!@!BB 8 2$78 8
 %%+*M*MOOF,,==>+
 $**400%%7--///d1g}5551gmmDGMM2G'#@A$$CYY,,S1F9944%mDKK&EF	 5   &B&&//0$w?D/BdFKK	C		5 		5	N 	N	 		 ##II((,,ZZ-P-U-U-WX''F & 5 5 7 9fk$789'lnMM''II((,,4;;6;;= F6v66  Xy1  8e k$678 (lnMM_$II((,,ZZ-P-U-U-WXF 4 4 4 !Xy2DE! 9f k$789 (lnMM_$II((,,ZZ-P-U-U-WXF 4 4 4 !VX/AB! 9f k$789 (lnMMZII((,,ZZ-P-U-U-WX''v44677YII((,,ZZ-P-U-U-WXv33566\!II((,,4;;6;;= 6v66OI9NOPPYII((,,ZZ-P-U-U-WX3v33Lj6KLMMV^''F =.55d;<='lnMMX''v||~..W_''F & 9fk$789'lnMMY''#**3v;77N"6EHH$7$79O9O#PQQ**,#**Q**,?  ]"#d)q.00##//##//&&22&&22##//! F|''/@@!$q'9+E+EFXPTUVPWHXXF1g002!#Sz222VT*33!"e&8&8999 DKK!>
K55222cBFT$Z'  ;;;$q'=1 99##F  c&k 23DG4N4N4PQD'0Q8R8R8T1U'V 	V"3i,]4;;-LMII== & #'* > 	 "(Q(B(B(D!EJ*,))*K*K"& 34-)$++tAw7Q7Q7ST  +L +' /. 11"5/$q'?31g++BII6a++-1g002 Ed1gYOPC[F9944%mDKK&EF	 5   ++6588#3#3#8#8#@#@AO+ 6588#3#3#8#8#I#IJ-- &//B":dkk4#@*MI11..r)D  F$$---))22489 "v}}? 
 .d337&r4v>>S8 8s   
r;;srI   r,   rM   List[VariableTracker]rN   Dict[str, VariableTracker]rs   r'   )F)r5  r   __qualname__r'   _nonvar_fieldsr;   r9  rB   r   Modulerv   r{   r   r   r   r   r   r   r   r   r   r   r   r   __classcell__rX   s   @r7   ro   ro   }   s    	
 
	'	'N22-02:?((//2	2:- )
@96 9c 9FW 9/* 
 `DDi &i -	i
 
ib x? &	x?
 -x? 
x? x?rm   ro   c                        e Zd Zdddhej                  Z	 d fdZd Zd Zd Ze	 e
j                  d      d	               Z fd
Z	 	 	 	 	 	 	 	 ddZ	 	 ddZ	 	 	 	 	 	 d fdZddZd fdZddZ xZS )r   
value_typeis_state_mutatedrr   Nc                 *   t        |      t        j                  j                  j                  u rt        d      d|v r+t        |d   dd       }t        |      |u rt        |      |d<   t        |    dd|i| d| _	        | j                  | _        y )NzScriptModules aren't supported in UnspecializedNNModuleVariable becuase their .forward function isn't a static member of their typerT  r   r4  Fr2   )r;   rB   jit_scriptRecursiveScriptModuler   r   ru   rv   rU  rZ   rr   )rw   r4  rN   lazy_value_to_becomerX   s       r7   rv   z&UnspecializedNNModuleVariable.__init__  s    ;%))++AAAW  6!#*6,+?RV#W E{22
 (,E{|$/u// % '+kk#rm   c                 <    t        |t              st        |      S |S r1   )r<   r   rw   attr_sources     r7   _wrap_sourcez*UnspecializedNNModuleVariable._wrap_source$  s    +'BC.{;;rm   c                 6    | j                   xs | j                  S r1   ry   rz   s    r7   r{   z8UnspecializedNNModuleVariable.get_nn_module_stack_source)  r|   rm   c                     || _         y r1   r~   r   s     r7   r   z8UnspecializedNNModuleVariable.set_nn_module_stack_source,  r   rm   c                  b   t         j                  j                  j                  t         j                  j                  j                  h} t         j                  j                  j
                  j                         D ch c]'  }t        |d      r|| vrt        |j                        ) c}S c c}w )N__code__)
rB   r   rP  __setattr__rv   ra   r  rJ   idrb  )	supportedrF   s     r7   _nn_module_method_idsz3UnspecializedNNModuleVariable._nn_module_method_ids/  s|     XX__00%((//2J2JK	 XX__--446
q*%!9*< qzzN
 	
 
s   =,B,c                    	 t        j                  | j                  d      }|t
        j                  j                  j                  t
        j                  j                  j                  t
        j                  j                  j                  fv r6|j                  t        j                  |      | gi       j                  |      S t        | 9  |      S # t        $ r}t        |d }~ww xY w)N__iter__)r   r   rT  r   NotImplementedErrorrB   r   r   rh  r   r   r   r
   r   r   ru   )rw   rI   r   erX   s       r7   r   z1UnspecializedNNModuleVariable.unpack_var_sequence:  s    	-''DB HH((HH""++HH((
 
 11..r2  "!"%& w*2..'  	-%1,	-s    C 	C*C%%C*c                    | j                   }t        |      r+|j                  |j                  | _        t	        ||||       d}t        | j                  |      }|t        j                  j                  j                  u rd|j                  vrt        j                  |d      }t        |t        j                        r]|j                   }| j#                  |d      j%                         j'                         s"| j#                  |d      j%                         j'                         s| j#                  |d      j%                         j'                         s| j#                  |d      j%                         j'                         s|j#                  |d      j'                         sx|j#                  |d      j'                         sX|j#                  |d	      j'                         s8|j#                  |d
      j'                         sd}| j                  j(                  }| j*                  r!t-        t-        | j*                  d      |      }	nd }	t/        | j*                  |       | j*                  r.t1        t3        t5        |            | j7                         ||      n	t9               }
|
5  t;        j<                  ||	      j?                  || gtA        |      z   |      cd d d        S # 1 sw Y   y xY w)Nr   r^   _backward_hooks_backward_pre_hooks_forward_hooks_forward_pre_hooks_global_backward_pre_hooks_global_backward_hooks_global_forward_hooks_global_forward_pre_hooksrX   r   )!r4  r   r   rT  rT   r   rB   r   rP  r   ra   r   r   r<   rc   r   nn_modules_globals_vtr   realizer  r^   rZ   r   rl   r\   r9  rd  r{   r   r
   r   r   r?   )rw   rI   rM   rN   rL   rW   r   forward_method
globals_vtrZ   ctxs              r7   r   z+UnspecializedNNModuleVariable.call_functionR  sZ    jj#  ,"%"3"3"2sD&9T__d+ +++	0M$33CCN.%*<*<=55
$$R):;CCEIIK'',ABJJLPPR'',<=EEGKKM'',@AIIKOOQ!--b2NOSSU!--b2JKOOQ!--b2IJNNP!--b2MNRRT$D00B;;
4;; DdKFF.t{{C@ {{ #BsGd==?S  	  	11"VDRRTFT$Z'	 	 	s   *5K))K2c                     
  fd}fd
fd
|t         j                  j                  j                  u r j                  rGj
                  j                  j                  t         j                  d      j                                 |d      d   } 
 |      }t        t        j                  |      j                               }	t        j                  |	t!                     S t#        d      )	Nc                     t        	j                        } t        j                  |      j                  D cg c]  }|j                          c}i j                         D ci c]  \  }}||j                          c}}}|j                          |j                  }| D ci c]  }|||   
 c}S c c}w c c}}w c c}w r1   )	r   r4  r   r  r  r	  r>   r
  r  )
r  r   rF   rG   rH   r  rM   rN   rW   rw   s
         r7   r  zIUnspecializedNNModuleVariable.trace_supported_methods.<locals>.get_kwargs  s    T*B3**2.33378a1'')89?HA1a**,,HJ %%'#--J.34Az!}$44 9H 5s   B9(B>)Cc                 R   | j                  d      j                         j                  }t        |t              sJ t        |j                               }|D cg c]  }|j                          }}|D cg c]  }t        |t        j                        r|  }}|S c c}w c c}w )Nr   )	r   ru  r>   r<   r=   r?   r  r
   r1  )
module_varparams_dictparams_listrD  rI   s       r7   get_current_parameterszUUnspecializedNNModuleVariable.trace_supported_methods.<locals>.get_current_parameters  s    $00]CKKMSSKk4000{1134K8CDu5==?DKD )eY%=%=> K 
  Es   B8#B$c                    g }t        | t              sJ  |       }| j                  d      j                         }|rI|j                  j                         D ],  }t        |t              sJ |j                   ||             . |S )Nr   )r<   r   r   ru  r>   r  extend)r|  r   r~  modules_dictsubmodule_varcollect_parametersr  rI   s        r7   r  zQUnspecializedNNModuleVariable.trace_supported_methods.<locals>.collect_parameters  s    Kj*GHHH0<K%11"jAIIKL%1%7%7%>%>%@ SM%m5RSSS&&'9-'QRS rm   r   r   )r   r  zMDiscrepancy between is_supported_nn_module_method and trace_supported_methods)rB   r   rP  r'  rZ   r   r6  r7  r   rW   r?   r=   fromkeysr   r
   r   r%   AssertionError)rw   rI   methodrW   rM   rN   r  r   r~  deduplicated_paramsr  r  s   `` ```    @@r7   trace_supported_methodsz5UnspecializedNNModuleVariable.trace_supported_methods  s    	5			 UXX__///{{		,,00t{{M:??A !+I6G,T7CK #'t}}['A'F'F'H"I11#<>  !_ rm   c                    |dv rzt        | j                  |      }| j                  r!t        t        | j                  d      |      }nd }t	        j
                  ||      j                  || gt        |      z   |      S |t        | j                  di       vr	 t        j                  t        | j                        |      }| j                  |      r| j                  |||||      S t        |t               r\t        t        t        | j                  d      |      d      }|j#                  t	        j
                  |j$                  |      ||      S t'        |d      r3t)        |j*                        | j-                         v rt/        d|        |dk(  rd	| j                  j0                  v r|d
   j3                         }|d   }	t        |	t        j4                        r*|	j7                         t8        j:                  j<                  u s|| j                  j0                  d	   v rd| _        no|| j                  j0                  d   v rd| _        nLt        |	t        j@                  t        jB                  f      s|| j                  j0                  d   v rd| _        |t8        j:                  jD                  jF                  u rjt        |d   t        jH                        rM|j#                  t	        j
                  t8        j:                  jD                  jJ                        | |d
   g|      S tL        
|   ||||      S # t        $ r d }Y w xY w)Nr   rX   r   ra   r_   rb  z&UnspecializedNNModuleVariable missing rc  r   r   r$   Tr   r   )(r   rT  rZ   r   r
   r   r   r?   r4  r   r   r;   r   is_supported_nn_module_methodr  r<   r   r   r_   rJ   rd  rb  rf  r   ra   r	  r1  r   rB   r   	ParameterrU  ro   r   rP  rc  r   __delattr__ru   r   )rw   rI   rW   rM   rN   r   rZ   r  	attr_namer4  rX   s             r7   r   z)UnspecializedNNModuleVariable.call_method  s    77$/B{{#Jt{{K$H$O11"VDRRTFT$Z'  wtzz:r:: //TZZ0@$G 11&933BdFSS&,/#z$++{CTJJ 55226??6R  
+v'4+E+E+GG FtfMN }$$**:M:M)M !G668	Q
 ui&>&>?))+uxx/A/AA$**"5"5m"DD,0D)$**"5"5j"AA,0D)%66%CC !DJJ$7$7
$CC -1D)444Q22:
 5522588??3N3NO47O  w"2tT6::O " s   )M MMc                 ~    | j                  ||      }t        |t        j                        r|j	                  |      S y r1   )r   r<   r
   ConstDictVariablemaybe_getitem_const)rw   rI   fieldname_vtdict_vts        r7   getattr_helperz,UnspecializedNNModuleVariable.getattr_helper  s8    ""2u-gy::;..w77rm   c                 
   |dv r|j                   j                  j                  | |      s| j                  j                  j                  d      rt        | j                  |      }t        |t              rmt        |      dk(  r_| j                  r>t        | j                  |      }t        |j                  t        j                               t!        j"                  i       S | j                  r|dv r|j                   j                  j                  | |      st        | j                  |      }t        | j                  |      t        j                  t        j$                               |j                   j&                  j)                  j+                                fdt        fdt-        |j/                               D              }t!        j"                  |t1        |            S t2        | i  ||      S )N)rl  rm  rn  ro  r   r   )ro  rn  c                     t         j                  j                  |      }t        |       }t	        |      }t        j                  ||      }||fS r1   )r
   r   r   r   r   r)   )irG   rH   r   
source_keysource_valuer4  hooks_dict_sources          r7   build_key_valuezBUnspecializedNNModuleVariable.var_getattr.<locals>.build_key_valueN  sQ    0077: 00A1E
,->
K+221lCEz!rm   c              3   >   K   | ]  \  }\  }} |||        y wr1   r2   )r4   r  rG   rH   r  s       r7   r8   z<UnspecializedNNModuleVariable.var_getattr.<locals>.<genexpr>Y  s'      -6QA1a(s   r   )r   r   r   r4  r   r   r   r<   r=   r  rZ   r   r   re   r   EMPTY_NN_MODULE_HOOKS_DICTr
   r  SEQUENCE_LENGTHr6  r7  rW   r   r>   r;   ru   r   )	rw   rI   rW   
hooks_dicthooks_sourcer   r  r  rX   s	         @@r7   r   z)UnspecializedNNModuleVariable.var_getattr#  s    
 
 99))FFd**''223MN$TZZ6
j$/C
Oq4H{{'1$++t'D%(33 , G G
 %66r:: KK
 II**GGdS T2J *4;; =+66|7S7STUII((,,->-C-C-EF	"  :CJDTDTDV:W F ..Z(1B  w"2t,,rm   c                     t        j                  |      }| j                  |d|      }|| j                  |d|      }|| j                  |d|      }|t        t        ||        |S )a  
        Dynamo tracing of nn.Module __getattr__ can be expensive if the model
        has deep submodule hierarchy. Since the __getattr__ is stable, we can
        directly look into the underlying datastructures. This saves a lot of
        compilation time.
        r   r   r   )r
   r   r  r   r   )rw   rI   rW   r  r   s        r7    manually_trace_nn_module_getattrz>UnspecializedNNModuleVariable.manually_trace_nn_module_getattrb  sp     ,,T2!!"mW=;%%b*g>C;%%b*g>C;$^R>
rm   rs   N)rI   r,   rM   rL  rN   rM  rs   r'   rJ  rK  )r5  r   rN  r+   rO  rv   r^  r{   r   r   rg   	lru_cacherf  r   r   r  r   r  r   r  rQ  rR  s   @r7   r   r     s      
#	1	1	N22
:- Y
  
/03#3 &3 -	3
 
3j6)6p\; &	\;
 -\; 
\;|=-~rm   r   c                       e Zd ZdZd Zy)$UnspecializedBuiltinNNModuleVariablezg
    Differentiates between builtin nn modules (e.g. torch.nn.Linear) and user defined nn modules.
    c                 <    t        |t              st        |      S |S r1   )r<   r   r\  s     r7   r^  z1UnspecializedBuiltinNNModuleVariable._wrap_sourcey  s    +'IJ5kBBrm   N)r5  r   rN  __doc__r^  r2   rm   r7   r  r  t  s    rm   r  c                   *     e Zd ZdZd fdZd Z xZS )FSDPManagedNNModuleVariablea  
    Tracing behavior: trace into submodules and treat them as Unspecialized, do not
    register parameters to the top-level, treat them as function inputs.

    Guards behavior: if 'skip_fsdp_guards', many guards that would be installed
    by a vanilla UnspecializedNNModuleVariable are simply dropped, on the basis
    that a user wrapping their model in FSDP(model) is already opting into a
    requirement to not modify internal model state, which would already break FSDP without
    compilation.
    c                 l    |j                  dd       }|J d       t        |   dd|i| || _        y )NrZ   zMFSDPManagedNNModule depends on having an accurate source to control guarding.r4  r2   )getru   rv   rZ   )rw   r4  rN   rZ   rX   s       r7   rv   z$FSDPManagedNNModuleVariable.__init__  sI    Hd+	[Z	[ 	/u//rm   c                     t        |t        t        f      s:t        j                  j
                  j                  rt        |      S t        |      S |S r1   )r<   r   r   rB   _dynamoconfigskip_fsdp_guardsr\  s     r7   r^  z(FSDPManagedNNModuleVariable._wrap_source  sG    ,.IJ
 }}##44)+662;??rm   r  )r5  r   rN  r  rv   r^  rQ  rR  s   @r7   r  r    s    	rm   r  rJ  )Frg   r   r  rc   
contextlibr   r   typingr   r   r   torch.nnrB   r  r	   r
   excr   r   r   r   guardsr   r   mutation_guardr   rZ   r   r   r   r   r   r   r   utilsr   r   r   r   r   r   r   r    r!   r"   r#   r   r%   r&   r'   	functionsr(   lazyr)   listsr*   user_definedr+   torch._dynamo.symbolic_convertr,   rT   r9  r   rP  r\   rl   ro   r   r  r  r2   rm   r7   <module>r     s        2 , ,  %  1 .      9 8 3 %   3 D;8 +s +UXX__ + +B{	? {	?|v$= vr+H "? rm   