
    sg*                         d dl Z d dlmZ d dlmZmZ d dlmZmZm	Z	m
Z
mZmZmZmZ d dlZd dlmZ e G d d             Z G d de      Z G d	 d
      Z G d d      Zy)    N)contextmanager)	dataclassfield)AnyCallableDictList
NamedTupleOptionalSetTuplec                       e Zd ZU dZ eej                  j                        Zej                  j                  e	d<   dZ
eeeef      e	d<   y)TracingConfiga  
    This represents a symbolic tracing configuration.

    Args:
        tracer (torch.fx.Tracer): An instance of :class:`torch.fx.Tracer` to
            use for symbolic tracing. The default value is the native
            :class:`torch.fx.Tracer` constructed with default arguments.
            However, the user may want to pass a different value such as the
            ``HFTracer`` for models in the HuggingFace Transformers_ library.
            .. _Transformers: https://huggingface.co/docs/transformers/index
        concrete_args (Optional[Dict[str, Any]]): Concrete arguments that
            should not be treated as ``torch.fx.Proxy`` when tracing the
            module ``forward()``. Passing ``concrete_args`` allows partially
            specializing the forward, e.g. to remove control flow or data
            structures. This ``concrete_args`` here is the same argument used
            in :meth:`~torch.fx.Tracer.trace`.
    )default_factorytracerNconcrete_args)__name__
__module____qualname____doc__r   torchfxTracerr   __annotations__r   r   r   strr        V/var/www/html/venv/lib/python3.12/site-packages/torch/distributed/fsdp/_trace_utils.pyr   r      s?    $ $EHHOODFEHHOOD.2M8DcN+2r   r   c                   ^    e Zd ZU dZej
                  ed<   eee	ej                  f      ed<   y)_ParamUsageInfoa6  
    This is used for ``_ExecutionInfo.module_to_param_usage_infos`` to record
    execution information. The ``dict`` maps modules to a list of these
    ``_ParamUsageInfo`` instances, where each instance represents a group of
    parameters used together.

    Specifically, for each module key in the ``dict``, each instance of this
    class represents either:
    (1) the module and some sublist of its ``named_parameters()`` used
    together in execution (see ``_patched_create_proxy()``), or
    (2) a submodule and all of ``submodule.named_parameters()`` (see
    ``_patched_call_module()``).

    Type (1) corresponds to directly using parameters in ops without calling
    ``forward()``, and type (2) corresponds to calling ``forward()``. The
    mapped-to lists in the ``dict`` follow the execution order.
    modulenamed_paramsN)r   r   r   r   nnModuler   r	   r   r   	Parameterr   r   r   r    r    #   s,    $ IIuS",,./00r   r    c                   4    e Zd ZdZdej
                  ddfdZy)_ExecutionInfoa1  
    This represents the execution order information from the forward pass.

    Attributes:
        curr_module (nn.Module): Current module being traced.
        module_forward_order (List[nn.Module]): The modules in (pre-)forward
            order, i.e. the order in which their ``forward()`` methods are
            called. Each call to a module's ``forward()`` corresponds to one
            element in the list.
        module_to_param_usage_infos (Dict[nn.Module, List[_ParamUsageInfo]]):
            Maps a module to a list of module execution infos. See
            :class:`_ParamUsageInfo` for details.
        param_forward_order (List[nn.Parameter]): The parameters in forward
            execution order, where only a parameter's first participation is
            included.
        visited_params (Set[nn.Parameter]): The parameters visited so far
            during the trace. This is only used during tracing for fast
            membership check. Invariant: The parameters in
            ``param_forward_order`` are exactly those in ``visited_params``.
    root_modulereturnNc                 `    || _         |g| _        |g i| _        g | _        t	               | _        y N)curr_modulemodule_forward_ordermodule_to_param_usage_infosparam_forward_ordersetvisited_params)selfr(   s     r   __init__z_ExecutionInfo.__init__P   s8    &16A]!T
( 8: 14r   )r   r   r   r   r#   r$   r3   r   r   r   r'   r'   :   s    *7BII 7$ 7r   r'   c                      e Zd ZddZedej                  j                  dej                  fd       Z
deded	ej                  d
edeedf   deeef   defdZ	 	 	 ddededeeej$                  f   dedej                  j&                  j(                  deedf   deeef   dee   dee   deeej                  j,                  gej                  j.                  f      dej                  j.                  fdZy)_ExecOrderTracerr)   Nc                     d | _         y r+   )	exec_info)r2   s    r   r3   z_ExecOrderTracer.__init__[   s	    37r   r   r(   c              #     K   t        |      | _        |j                  }|j                  }t	        j
                  | j                  || j                        |_        t        |j                               }t	        j
                  | j                  || j                  |      |_        	 d  ||_        ||_        y # ||_        ||_        w xY wwr+   )
r'   r7   call_modulecreate_proxy	functoolspartial_patched_call_moduledictnamed_parameters_patched_create_proxy)r2   r   r(   orig_call_moduleorig_create_proxyfqn_to_params         r   patch_tracerz_ExecOrderTracer.patch_tracer^   s     '4!--"//&..%%'7
 K88:;'//&&NN	
	4!1F"3F "2F"3Fs   B#C&B9 *C9C		Cr9   r7   r!   forwardargs.kwargsc                 l   |j                   j                  |       t        |j                               }|j                  }|rG||j
                  v sJ d       |j
                  |j                     j                  t        ||             |}	||_        g |j
                  |<    |||||      }
|	|_        |
S )a  
        Overrides ``call_module`` to save execution information to
        ``exec_info``. Note that ``call_module`` is called during symbolic
        tracing for each non-root module.

        Args:
            call_module (Callable): Original ``call_module`` to override.
            exec_info (_ExecutionInfo): Used to record execution information.
            module (nn.Module): Module corresponding to this ``call_module``.
            forward (Callable): ``forward()`` method of ``module`` to be called
                for this ``call_module``.
            args (Tuple[Any, ...]): Positional arguments for ``forward``.
            kwargs (Dict[str, Any]): Keyword arguments for ``forward``.

        Returns:
            Same return value as ``call_module``.
        zPThe current module should have already been processed by a patched `call_module`)r-   appendlistr?   r,   r.   r    )r2   r9   r7   r!   rE   rF   rG   r"   r,   prev_curr_moduleoutputs              r   r=   z%_ExecOrderTracer._patched_call_modules   s    6 	&&--f5F3356++yDDDbabD11)2G2GHOO5 ' &	8:	--f5VWdF; 0	r   r:   rC   kindtargetname	type_exprproxy_factory_fnc           	      d    |||||||	|
      }|j                   }|dv r|g }|D ]  }t        |t        j                  j                        s(|j
                  j                  |v sA||j
                  j                     }|j                  |j
                  j                  |f       ||j                  vs|j                  j                  |       |j                  j                  |        |r(|j                  |   j                  t        ||             |S |dk(  rt        |j                               }|r(|j                  |   j                  t        ||             |D ]J  \  }}||j                  vs|j                  j                  |       |j                  j                  |       L |S )a  
        Overrides ``create_proxy`` to save execution information to
        ``exec_info``. Note that ``create_proxy`` is called during symbolic
        tracing for each leaf function/method/module.

        Args:
            create_proxy (Callable): Original ``create_proxy`` to override.
            exec_info (_ExecutionInfo): Used to record execution information.
            fqn_to_param (Dict[str, nn.Parameter]): ``dict`` version of the
                root module's ``named_parameters()`` with FQN as key and
                parameter as value.
            kind (str): Kind of the target method ('call_function',
                'call_method', 'get_attr', 'call_module', 'placeholder', or
                'output'). See :class:`torch.fx.Graph` for details. This is
                passed to ``create_proxy``.
            target (torch.fx.node.Target): Contains the string name of the
                function/method/module. This is passed to ``create_proxy``.
            args (Tuple[Any, ...]): Positional arguments for the function/
                method/module. This is passed to ``create_proxy``.
            kwargs (Dict[str, Any]): Keyword arguments for the function/method/
                module. This is passed to ``create_proxy``
            name (Optional[str]): An optional string name for the ``Node``
                created in ``create_proxy``. This is passed to
                ``create_proxy``.
            type_expr (Optional[Any]): An optional type annotation representing
                the Python type that the output of the node has. This is passed
                to ``create_proxy``.
            proxy_factory_fn (Callable[[torch.fx.Node], torch.fx.Proxy]):
                An alternative proxy constructor used in ``create_proxy``. This
                is passed to ``create_proxy``.

        Returns:
            torch.fx.Proxy: Created ``Node`` wrapped in a ``Proxy`` object.
        )call_functioncall_methodr9   )r,   
isinstancer   r   ProxynoderN   rI   r1   addr/   r.   r    rJ   r?   )r2   r:   r7   rC   rM   rN   rF   rG   rO   rP   rQ   proxyr,   r"   argparam_s                    r   r@   z&_ExecOrderTracer._patched_create_proxy   s   ` &$i9I
  ++33?A 	HC"37HHOO|; ,SXX__ =$++SXX__e,DE 	(@(@@%4488?%99@@G	H  99+FMM'\B  ]" < < >?L55kBII#K> ) @5	 8 88,,0071188?@ r   )r)   N)NNN)r   r   r   r3   r   r   r   r   r#   r$   rD   r   r'   r   r   r   r   r=   r%   rW   Targetr   NoderV   r@   r   r   r   r5   r5   Z   sx   8 4588?? 4 4 4(** "*
 		* * CHo* S#X* 
*l ##'PTOO "O 3,-	O O $$O CHoO S#XO smO C=O #8UXX]]OUXX^^,K#LMO 
Or   r5   )r;   
contextlibr   dataclassesr   r   typingr   r   r   r	   r
   r   r   r   r   torch.nnr#   r   r    r'   r5   r   r   r   <module>rc      sY     % ( N N N   3 3 3.1j 1.7 7@T Tr   