
    sgLz                     r   d dl Z d dlZd dlZd dlZd dlmZmZmZmZm	Z	m
Z
mZmZmZmZmZmZ d dlZd dlmc mZ d dlmZmZmZmZ d dlmZ d dlmZ d dlmZ d dlm Z  d dl!m"Z" d d	l#m$Z$ d
gZ%eeedf   ef   Z& edd      Z' ejP                         Z)dZ* e jV                  de*      Z,dZ-d Z.d Z/d Z0de	ejb                  e"      fdZ2de3ddfdZ4 G d dej@                        Z5 G d d
e5      Z6d Z7d Z8d Z9d Z: ejv                  e6e8        ejv                  ejx                  jz                  e:       y)     N)AnyCallableDictIteratorListMappingOptionalSetTupleTypeTypeVarUnion)devicedtypennTensor)_remote_device)instantiator)_internal_rpc_pickler)Module)	Parameter)RemovableHandleRemoteModule.Tr   )bound)onr   is_device_map_setis_scriptablegenerated_methodsmodule_rref_SerializedRemoteModule)training_parameters_buffers_non_persistent_buffers_set_backward_hooks_backward_pre_hooks_is_full_backward_hook_forward_hooks_forward_hooks_with_kwargs_forward_hooks_always_called_forward_pre_hooks_forward_pre_hooks_with_kwargs_state_dict_hooks_state_dict_pre_hooks_load_state_dict_pre_hooks_load_state_dict_post_hooksr/   _modulesforward_asyncforwardc                 0    t        j                  | |       y N)r   -instantiate_scriptable_remote_module_template)module_interface_cls!enable_moving_cpu_tensors_to_cudas     Y/var/www/html/venv/lib/python3.12/site-packages/torch/distributed/nn/api/remote_module.py_instantiate_templater;   V   s    >>?    c                      | |i |}t        |t        j                        st        dt	        |       d      |j                  |       |S )NzmExpect `module_cls(*args, **kwargs)` returns an instance of <class nn.Module>, but it returns an instance of .)
isinstancer   r   
ValueErrortypeto)
module_clsargskwargsr   modules        r:   _create_modulerG   \   sT    ((Ffbii(--1&\N!=
 	
 IIfMr<   c                     t        | |||      }|t        j                  j                  |      }t	        j
                  ||      S r6   )rG   torchjitscriptrpcRRef)rC   rD   rE   r   r8   rF   s         r:   _create_module_with_interfacerN   g   s@     Jff=F'!!&)88F011r<   returnc                     g }| j                         j                  |      D ]&  }|j                  t        j                  |             ( |S r6   )local_value
parametersappendrL   rM   )r    recurseretparams       r:   _param_rrefsrW   p   sC    %'C((*55g> $

388E?#$Jr<   namec                      t        d|  d      )Nz	Method ``z!`` not supported for RemoteModuler@   )rX   s    r:   _raise_not_supportedr[   w   s    
y&GH
IIr<   c                       e Zd Z fdZ	 	 	 dMdedeej                     dee	   dee
eef      def
 fdZdNd	ed
eej                   e      fdZd
ej                   ej                     fdZej*                  j,                  d        Zej*                  j,                  d        Z	 dNdedee   ded
dfdZdedee   d
dfdZdedee   d
dfdZdedeegdf   d
efdZdOdedee e!e"f      d
efdZ#dOdedee e!e"f      d
efdZ$dOdedee e!e"f      d
efdZ%ded
efdZ&dede e'ef   d
efd Z(ded
efd!Z)ded
efd"Z*ded
efd#Z+ded
efd$Z,d
efd%Z-d&eee.e.ge de.f   f   d
e/fd'Z0	 	 dPd&e eee	ed(f   gee   f   eee	ed(f   e
eef   gee	ee
eef   f      f   f   d)ed*ed
e/fd+Z1	 	 dPd&e eee	ed(f   egee   f   eee	ed(f   e
eef   egee   f   f   d)ed*ed
e/fd,Z2d- Z3	 	 dQd.e4eef   d/ed0efd1Z5dNd	ed
e6e   fd2Z7	 dRd3ed	ed4ed
e6e	eef      fd5Z8dNd	ed
e6e   fd6Z9	 dRd3ed	ed4ed
e6e	eef      fd7Z:d
e6e   fd8Z;d
e6e	eef      fd9Z<d
e6e   fd:Z=	 	 	 dSd;ee>e      d3ed4efd<Z?dNded=ed
efd>Z@ded
efd?ZAdNded@ed
efdAZBdNdBed
dfdCZCded
efdDZDd
efdEZEdFed
efdGZFdH ZGdI ZHdJ ZIeJ	 dOdedKej                   ej                     defdL       ZK xZLS )T_RemoteModulec                 `    t         j                  j                  d       t        |   |       S )Nz&torch.distributed.nn.api.remote_module)rI   _C_log_api_usage_oncesuper__new__)clsrD   rE   	__class__s      r:   rb   z_RemoteModule.__new__|   s%    $$%MNws##r<   Nremote_devicerC   rD   rE   _module_interface_clsc           	         t         |           | j                  |      }||nd}||ni }|d| _        t	        j
                  | j                  t        ||f      }| j                  ||       t	        j
                  | j                  t        ||f      }|j                          t	        j                  | j                  t        |||| j                  |f      | _        nTd| _        t        j                  | _        t	        j"                  | j                  t$        |||| j                  f      | _        | j'                          | j)                          y)a  
        RemoteModule instance can only be created after RPC initialization.

        It creates a user-specified module on a specified remote node.
        It behaves like a regular ``nn.Module`` except that the ``forward`` method is
        executed on the remote node.
        It takes care of autograd recording to ensure the backward pass propagates
        gradients back to the corresponding remote module.
        It can be shared across processors using `RPC framework <https://pytorch.org/docs/stable/rpc.html>`__,
        without incurring any overheads of copying the actual module,
        which is equivalent to an :class:`~torch.distributed.rpc.RRef`
        pointing to the remote module.

        The arguments of ``forward_async`` and ``forward`` are the same as
        the ``forward`` method of the module returned by the ``module_cls``.

        Apart from ``forward_async`` and ``forward``, no other methods are supported from nn.Module for now.

        Particularly, to create a hybrid model, typically the local modules should be
        created outside of remote modules, rather than as submodules of any remote module (by calling ``add_module``).
        Hybrid Example:
                >>> class HybridModel(nn.Module):
                >>>     def __init__(self) -> None:
                >>>         nn.Module.__init__(self)
                >>>         self.remote_embedding = RemoteModule(...)
                >>>         self.local_linear = nn.Linear(...)

        For example, if ``module_cls`` returns an instance of ``nn.Linear``,
        that has ``forward`` method signature, ``def forward(input: Tensor) -> Tensor:``,
        the generated ``RemoteModule`` will have 2 methods in signature of
        ``def forward(input: Tensor) -> Tensor:`` and
        ``def forward_async(input: Tensor) -> Future[Tensor]:``.

        .. note::
            If the remote module is placed on a cuda device,
            any input CPU tensors will be automatically moved to the same cuda device,
            and GPU tensors are returned over the wire according to the device map of the remote worker on TensorPipe RPC backend.

        Args:
            remote_device (str): Device on the destination worker where we'd like to place this module.
                The device can be a local device or a remote device specified by one of the following remote
                formats:

                    1. "rank:<rank>/<device>" (ex: "rank:0/cuda:0").
                    2. "<worker_name>/<device>" (ex: "trainer0/cuda:0").

                In addition, the device field can be optional and the default value is "cpu".
            module_cls (nn.Module): For example,
                >>> class MyModule(nn.Module):
                >>>     def forward(input):
                >>>         return input + 1
                >>>
                >>> module_cls = MyModule
            args (Sequence, optional): args to be passed to ``module_cls``.
            kwargs (Dict, optional): kwargs to be passed to ``module_cls``.
            _module_interface_cls (type, optional): The TorchScript interface type for the module
                to be created. The type object should be decorated by @torch.jit.interface.
                If not provided, the generated RemoteModule is not torchscript-able.
                Warning, this is an experimental API and susceptible to frequent changes.

        Returns:
            A remote module instance which wraps the :class:`~nn.Module` created by the
            user-provided ``module_cls``, it has a blocking ``forward`` method and an
            asynchronous ``forward_async`` method that returns a future of the ``forward`` call
            on the user-provided module on the remote side.

        Example::
            Run the following code in two different processes:

            >>> # xdoctest: +SKIP("distributed")
            >>> # On worker 0:
            >>> import torch
            >>> import torch.distributed.rpc as rpc
            >>> from torch import nn, Tensor
            >>> from torch.distributed.nn.api.remote_module import RemoteModule
            >>>
            >>> rpc.init_rpc("worker0", rank=0, world_size=2)
            >>> remote_linear_module = RemoteModule(
            >>>     "worker1/cpu", nn.Linear, args=(20, 30),
            >>> )
            >>> input = torch.randn(128, 20)
            >>> ret_fut = remote_linear_module.forward_async(input)
            >>> ret = ret_fut.wait()
            >>> rpc.shutdown()

            >>> # On worker 1:
            >>> import torch
            >>> import torch.distributed.rpc as rpc
            >>>
            >>> rpc.init_rpc("worker1", rank=1, world_size=2)
            >>> rpc.shutdown()
        N TF)ra   __init___prepare_initr   rL   	rpc_asyncr   r;   _init_templatewaitrpc_syncrN   r   r    $_NON_SCRIPTABLE_REMOTE_MODULE_MODULE_generated_methodsr   remoterG   _install_generated_methods_check_attribute_picklability)	selfre   rC   rD   rE   rf   r9   futrd   s	           r:   ri   z_RemoteModule.__init__   sC   H 	,0,>,>},M) 'tR!-2 ,!%D --%&(IJC %'H
 --%&(IJC HHJ
  #||-T64;;8MN D "'D4GG "  #zzT64;;7 D 	'')**,r<   rT   rO   c                 f    t        j                  | j                  t        | j                  |f      S )a  
        Return a list of :class:`~torch.distributed.rpc.RRef` pointing to the remote module's parameters.

        This can typically be used in conjunction
        with :class:`~torch.distributed.optim.DistributedOptimizer`.

        Args:
            recurse (bool): if True, then returns parameters of the remote
                module and all submodules of the remote module. Otherwise,
                returns only parameters that are direct members of the
                remote module.

        Returns:
            A list of :class:`~torch.distributed.rpc.RRef` (``List[RRef[nn.Parameter]]``)
            to remote module's parameters.
        )rD   )rL   rn   r   rW   r    rt   rT   s     r:   remote_parametersz_RemoteModule.remote_parameters  s&    " ||DGG\9I9I78STTr<   c                     | j                   S )zcReturn an :class:`~torch.distributed.rpc.RRef` (``RRef[nn.Module]``) pointing to the remote module.)r    rt   s    r:   get_module_rrefz_RemoteModule.get_module_rref0  s    r<   c                     t        d      )Nz]Cannot pickle RemoteModule in python pickler. RemoteModule can only be pickled when using RPCRuntimeErrorrz   s    r:   __getstate__z_RemoteModule.__getstate__4  s    k
 	
r<   c                     t        d      )NzaCannot unpickle RemoteModule in python pickler. RemoteModule can only be unpickled when using RPCr}   )rt   states     r:   __setstate__z_RemoteModule.__setstate__:  s    o
 	
r<   rX   tensor
persistentc                 B    t        | j                  j                         y r6   )r[   register_buffer__name__)rt   rX   r   r   s       r:   r   z_RemoteModule.register_buffer@  s     	T11::;r<   rV   c                 B    t        | j                  j                         y r6   )r[   register_parameterr   )rt   rX   rV   s      r:   r   z _RemoteModule.register_parameterE  s    T44==>r<   rF   c                 B    t        | j                  j                         y r6   )r[   
add_moduler   )rt   rX   rF   s      r:   r   z_RemoteModule.add_moduleH      T__556r<   rt   fnc                 B    t        | j                  j                         y r6   )r[   applyr   )rt   r   s     r:   r   z_RemoteModule.applyK      TZZ001r<   r   c                 B    t        | j                  j                         y r6   )r[   cudar   rt   r   s     r:   r   z_RemoteModule.cudaN      TYY//0r<   c                 B    t        | j                  j                         y r6   )r[   ipur   r   s     r:   r   z_RemoteModule.ipuQ      TXX../r<   c                 B    t        | j                  j                         y r6   )r[   xpur   r   s     r:   r   z_RemoteModule.xpuT  r   r<   c                 B    t        | j                  j                         y r6   )r[   cpur   rz   s    r:   r   z_RemoteModule.cpuW  r   r<   dst_typec                 B    t        | j                  j                         y r6   )r[   rA   r   )rt   r   s     r:   rA   z_RemoteModule.typeZ  r   r<   c                 B    t        | j                  j                         y r6   )r[   floatr   rz   s    r:   r   z_RemoteModule.float]  r   r<   c                 B    t        | j                  j                         y r6   )r[   doubler   rz   s    r:   r   z_RemoteModule.double`  s    T[[112r<   c                 B    t        | j                  j                         y r6   )r[   halfr   rz   s    r:   r   z_RemoteModule.halfc  r   r<   c                 B    t        | j                  j                         y r6   )r[   bfloat16r   rz   s    r:   r   z_RemoteModule.bfloat16f      T]]334r<   c                 B    t        | j                  j                         y r6   )r[   rB   r   rt   rD   rE   s      r:   rB   z_RemoteModule.toi  s    TWW--.r<   hookc                 B    t        | j                  j                         y r6   )r[   register_backward_hookr   )rt   r   s     r:   r   z$_RemoteModule.register_backward_hookl  s     	T88AABr<   .prependwith_kwargsc                 B    t        | j                  j                         y r6   )r[   register_forward_pre_hookr   rt   r   r   r   s       r:   r   z'_RemoteModule.register_forward_pre_hookq  s     	T;;DDEr<   c                 B    t        | j                  j                         y r6   )r[   register_forward_hookr   r   s       r:   r   z#_RemoteModule.register_forward_hook  s     	T77@@Ar<   c                 B    t        | j                  j                         y r6   )r[   
state_dictr   r   s      r:   r   z_RemoteModule.state_dict  r   r<   r   strictassignc                 B    t        | j                  j                         y r6   )r[   load_state_dictr   )rt   r   r   r   s       r:   r   z_RemoteModule.load_state_dict  s     	T11::;r<   c                     t        d      )Nz_Method ``parameters`` not supported for RemoteModule. Please use ``remote_parameters`` instead.rZ   rw   s     r:   rR   z_RemoteModule.parameters  s    m
 	
r<   prefixremove_duplicatec                 B    t        | j                  j                         y r6   )r[   named_parametersr   rt   r   rT   r   s       r:   r   z_RemoteModule.named_parameters  s     	T22;;<r<   c                 B    t        | j                  j                         y r6   )r[   buffersr   rw   s     r:   r   z_RemoteModule.buffers      T\\223r<   c                 B    t        | j                  j                         y r6   )r[   named_buffersr   r   s       r:   r   z_RemoteModule.named_buffers  s     	T//889r<   c                 B    t        | j                  j                         y r6   )r[   childrenr   rz   s    r:   r   z_RemoteModule.children  r   r<   c                 B    t        | j                  j                         y r6   )r[   named_childrenr   rz   s    r:   r   z_RemoteModule.named_children      T0099:r<   c                 B    t        | j                  j                         y r6   )r[   modulesr   rz   s    r:   r   z_RemoteModule.modules  r   r<   memoc                 B    t        | j                  j                         y r6   )r[   named_modulesr   )rt   r   r   r   s       r:   r   z_RemoteModule.named_modules  s     	T//889r<   modec                 R    | j                   j                         j                         S r6   )r    rn   train)rt   r   s     r:   r   z_RemoteModule.train  s     ((*0022r<   c                 R    | j                   j                         j                         S r6   )r    rn   evalrz   s    r:   r   z_RemoteModule.eval  s     ((*//11r<   requires_gradc                 B    t        | j                  j                         y r6   )r[   requires_grad_r   )rt   r   s     r:   r   z_RemoteModule.requires_grad_  r   r<   set_to_nonec                 B    t        | j                  j                         y r6   )r[   	zero_gradr   )rt   r   s     r:   r   z_RemoteModule.zero_grad  s    T^^445r<   c                 B    t        | j                  j                         y r6   )r[   share_memoryr   rz   s    r:   r   z_RemoteModule.share_memory  s    T..778r<   c                 B    t        | j                  j                         y r6   )r[   
extra_reprr   rz   s    r:   r   z_RemoteModule.extra_repr  r   r<   remote_device_strc                    t        j                         sJ d       t        |      }|j                         |j                         n|j	                         | _        t        |j                               | _        t        j                         }t        |j                  |j                  | j
                                    | _        t        j                  | j                        j                  dk(  }|S )zjPrepare the initialization and returns whether to enable automatically moving CPU tensors to CUDA devices.zRemoteModule only works in RPC.r   )rL   _is_current_rpc_agent_setr   worker_namerankr   strr   _get_current_rpc_agentbool_get_device_mapget_worker_infor   rI   rA   )rt   r   re   agentr9   s        r:   rj   z_RemoteModule._prepare_init  s     ,,.Q0QQ.&'89 ((*6 %%'##% 	
 -..01**, "&!!%"7"7"@A"
 -2LL,E,J,Jf,T)00r<   c                 R    t        j                  ||      }|j                  | _        y)z#Instantiate template on local side.N)r   r7   rp   r   )rt   r8   r9   generated_modules       r:   rl   z_RemoteModule._init_template  s)    'UU "C
 "2!D!Dr<   c                     | j                   j                         D ]"  }|t        vs|t        vst	        d| d       y)z]Check if all the attribute has explicitly defined whether to be pickled (i.e., picklability).z
Attribute zn must be either in ``_REMOTE_MODULE_PICKLED_ATTRIBUTES`` or ``_REMOTE_MODULE_ATTRIBUTES_IGNORE_FOR_PICKLING``.N)__dict__keys!_REMOTE_MODULE_PICKLED_ATTRIBUTES-_REMOTE_MODULE_ATTRIBUTES_IGNORE_FOR_PICKLINGAttributeError)rt   ks     r:   rs   z+_RemoteModule._check_attribute_picklability  sP    ##% 	A::JJ$  $I I 	r<   c           	          | j                   D ]N  }|j                  }t        j                  j	                  |      }t        | |t        j                  ||              P y r6   )r   r   rI   rJ   exportsetattrtypes
MethodType)rt   methodmethod_names      r:   rr   z(_RemoteModule._install_generated_methods  sO    ,, 	GF //KYY%%f-FD+u'7'7'EF	Gr<   r    c                    t         j                  t              }|j                  |       }|d|_        |j                  ||       nd|_        t        j                  |_        ||_	        |j                          |j                          |S )a&  
        Besides the constructor, a RemoteModule instance can also be initialized given a module RRef.

        This alternate initialization method can be particularly useful if we want to create multiple
        RemoteModule instances that share the same underlying module and reduce memory consumption.

        Moreover, this also provides a workaround for passing script RemoteModule over RPC,
        which is not supported. The recommended way is as follows:

            1. the sender creates a RemoteModule;
            2. the sender sends its ``module_rref`` over RPC;
            3. the receiver calls this method to initialize another RemoteModule using the same ``module_rref``.

        Example::
            Run the following code in two different processes:

            >>> # xdoctest: +SKIP("distributed")
            >>> # On worker 0:
            >>> import torch
            >>> import torch.distributed.rpc as rpc
            >>> from torch import nn, Tensor
            >>> from torch.distributed.nn.api.remote_module import RemoteModule
            >>>
            >>> rpc.init_rpc("worker0", rank=0, world_size=2)
            >>> remote_module = RemoteModule(
            >>>     "worker1/cpu", nn.Linear, args=(20, 30),
            >>> )
            >>>
            >>> remote_module1 = rpc.rpc_sync(
            >>>     "worker1/cpu",
            >>>     RemoteModule.init_from_module_rref,
            >>>     ("worker1/cpu", remote_module1.get_module_rref()),
            >>> )
            >>> rpc.shutdown()

            >>> # On worker 1:
            >>> import torch
            >>> import torch.distributed.rpc as rpc
            >>>
            >>> rpc.init_rpc("worker1", rank=1, world_size=2)
            >>> rpc.shutdown()

        Args:
            remote_device (str): Device on the destination worker where we'd like to place this module.
                The device can be a local device or a remote device specified by one of the following remote
                formats:

                    1. "rank:<rank>/<device>" (ex: "rank:0/cuda:0").
                    2. "<worker_name>/<device>" (ex: "trainer0/cuda:0").

                In addition, the device field can be optional and the default value is "cpu".
            module_rref (RRef[nn.Module]): The module reference shared by both the caller and
                the created remote module.
            _module_interface_cls (type, optional): The TorchScript interface type for the module
                to be created. The type object should be decorated by @torch.jit.interface.
                If not provided, the generated RemoteModule is not torchscript-able.
                Warning, this is an experimental API and susceptible to frequent changes.

        Returns:
            A remote module instance which wraps the :class:`~nn.Module` created by the
            user-provided ``module_rref``, it has a blocking ``forward`` method and an
            asynchronous ``forward_async`` method that returns a future of the ``forward`` call
            on the user-provided module on the remote side.
        TF)objectrb   r   rj   r   rl   ro   rp   r   r    rr   rs   )re   r    rf   remote_moduler9   s        r:   init_from_module_rrefz#_RemoteModule.init_from_module_rref  s    R |4,9,G,G,V) ,*.M'((%'H +0M'4GG + %0!002335r<   )NNN)Tr6   )FF)TF) TT)Nr   T)Mr   
__module____qualname__rb   r   r   r   r   r	   r   r   r   ri   r   r   rL   rM   r   rx   r{   rI   rJ   r   r   r   r   r   r   r   r   r   r   r   intr   r   r   r   r   r   rA   r   r   r   r   rB   _grad_tr   r   r   r   r   r   r   r   rR   r   r   r   r   r   r   r
   r   r   r   r   r   r   r   rj   rl   rs   rr   staticmethodr   __classcell__rd   s   @r:   r]   r]   {   s	   $ !%+/%)Z-Z- OZ- uo	Z-
 c3h(Z-  #Z-xU Uchhy>Q9R U& "))!4   YY
 

 YY
 
 GK<<!)&!1<?C<	<
?s ?8I3F ?4 ?7s 7HV,< 7 72A 28VHdN3 2 211 1huS&['9: 1a 10! 0XeCK&89 0Q 00! 0XeCK&89 0Q 00! 0 011 1eSj 1 1a 12A 2! 23Q 31 311 1 15q 5Q 5/Q /Cfgw7tW}9MMNC	C !FasCx)8C=89E#s(OT#s(^4sDcN2346
F F F 
F( !	BasCx#.=>asCx$sCx.#>MNP
	B 	B 	B 
	B7 	<CH%< < 	<
$ 
(92E 
 PT==)-=HL=	%Y'	(=
4t 4x/? 4 PT::)-:HL:	%V$	%:
5(6* 5;sF{); < ;4&) 4
 '+!%	:s6{#: : 	:3A 3T 3Q 321 2 2;Q ;t ;q ;6T 6T 691 9 97C 71s 1t 12E
G  &*]]XXbii(]  #] ]r<   r]   c                   j     e Zd ZdZ	 	 ddedeej                     dee	   dee
eef      f fdZ xZS )r   a  
        A RemoteModule instance can only be created after RPC initialization.

        It creates a user-specified module on a specified remote node.
        It behaves like a regular ``nn.Module`` except that the ``forward`` method is
        executed on the remote node.
        It takes care of autograd recording to ensure the backward pass propagates
        gradients back to the corresponding remote module.

        It generates two methods ``forward_async`` and ``forward`` based on the
        signature of the ``forward`` method of ``module_cls``. ``forward_async``
        runs asynchronously and returns a Future. The arguments of ``forward_async``
        and ``forward`` are the same as the ``forward`` method of the module
        returned by the ``module_cls``.

        For example, if ``module_cls`` returns an instance of ``nn.Linear``,
        that has ``forward`` method signature: ``def forward(input: Tensor) -> Tensor:``,
        the generated ``RemoteModule`` will have 2 methods with the signatures:

        | ``def forward(input: Tensor) -> Tensor:``
        | ``def forward_async(input: Tensor) -> Future[Tensor]:``

    Args:
        remote_device (str): Device on the destination worker where we'd like to place this module.
            The format should be "<workername>/<device>", where the device field can be parsed as torch.device type.
            E.g., "trainer0/cpu", "trainer0", "ps0/cuda:0".
            In addition, the device field can be optional and the default value is "cpu".
        module_cls (nn.Module): Class for the module to be created remotely. For example,

            >>> class MyModule(nn.Module):
            >>>     def forward(input):
            >>>         return input + 1
            >>>
            >>> module_cls = MyModule

        args (Sequence, optional): args to be passed to ``module_cls``.
        kwargs (Dict, optional): kwargs to be passed to ``module_cls``.

    Returns:
        A remote module instance which wraps the :class:`~nn.Module` created by the
        user-provided ``module_cls``, it has a blocking ``forward`` method and an
        asynchronous ``forward_async`` method that returns a future of the ``forward`` call
        on the user-provided module on the remote side.

    Example::
        Run the following code in two different processes:

        >>> # xdoctest: +SKIP("distributed")
        >>> # On worker 0:
        >>> import torch
        >>> import torch.distributed.rpc as rpc
        >>> from torch import nn, Tensor
        >>> from torch.distributed.nn.api.remote_module import RemoteModule
        >>>
        >>> rpc.init_rpc("worker0", rank=0, world_size=2)
        >>> remote_linear_module = RemoteModule(
        >>>     "worker1/cpu", nn.Linear, args=(20, 30),
        >>> )
        >>> input = torch.randn(128, 20)
        >>> ret_fut = remote_linear_module.forward_async(input)
        >>> ret = ret_fut.wait()
        >>> rpc.shutdown()

        >>> # On worker 1:
        >>> import torch
        >>> import torch.distributed.rpc as rpc
        >>>
        >>> rpc.init_rpc("worker1", rank=1, world_size=2)
        >>> rpc.shutdown()

        Furthermore, a more practical example that is combined with
        `DistributedDataParallel <https://pytorch.org/docs/stable/nn.html#torch.nn.parallel.DistributedDataParallel>`__ (DDP)
        can be found in this `tutorial <https://pytorch.org/tutorials/advanced/rpc_ddp_tutorial.html>`__.
    re   rC   rD   rE   c                 *    t         |   ||||       y r6   )ra   ri   )rt   re   rC   rD   rE   rd   s        r:   ri   zRemoteModule.__init__  s     	
D&Ar<   )NN)r   r   r   __doc__r   r   r   r   r	   r   r   r   ri   r  r  s   @r:   r   r   ]  sa    I^ !%+/BB OB uo	B
 c3h(B Br<   c            	         t         j                  |       }t        j                  t              }|j
                  j                  |j                                t        j                  j                  |j                        |_        |j                  D ]N  }|j                  }t        j                  j!                  |      }t#        ||t%        j&                  ||             P |S )zDeserializes a RemoteModule.)r!   _maker   rb   r   r   update_asdictrL   PyRRef_deserializer    r   r   rI   rJ   r   r   r   r   )remote_module_pickled_attrsserialized_remote_modulemr   r   s        r:   _remote_module_receiverr    s      7<<#  	|$AJJ.6689 JJ++AMM:AM %% =oo!!&); 0 0 ;<=
 Hr<   c                 &   i }| j                   j                         D ]T  \  }}|dk(  r|j                         ||<   |t        v r|||<   -|t        vs6t        d| dt        j                         V t        t        |j                               fS )zSerialize a RemoteModule.r    zThe new attribute ``z`` of RemoteModule is ignored during RPC pickling. To pickle this attribute, please add it to ``_REMOTE_MODULE_PICKLED_ATTRIBUTES``. Otherwise, please explicitly add it to ``_REMOTE_MODULE_ATTRIBUTES_IGNORE_FOR_PICKLING``.)file)r   items
_serializer   r   printsysstderrr  tuplevalues)r   pickled_attrsr   vs       r:   _remote_module_reducerr    s    M&&,,. 1 ||~M!33 M!CC&qc *l l ZZ	  	 m""$% r<   c                 n    t        j                  |       }t        j                  j	                  |      }|S )zQDeserializes a RecursiveScriptModule that does not contain a script RemoteModule.)ioBytesIOrI   rJ   load)"recursive_script_module_serializedfr  s      r:   !_recursive_script_module_receiverr#    s*     	

56A		qAHr<   c                     t        | j                  d      rt        d      t        j                         }t
        j                  j                  | |       t        |j                         ffS )zmSerialize a RecursiveScriptModule that does not contain a script RemoteModule, and raises an error otherwise.r    zPassing a script RemoteModule over RPC is not supported. Please create a RemoteModule in the sender, send the `module_rref` to the receiver, and create a new instance on the receiver end by passing this `module_rref`.)
hasattr_cr~   r  r  rI   rJ   saver#  getvalue)recursive_script_moduler"  s     r:    _recursive_script_module_reducerr*    sY    &))=9C
 	

 	

A	IINN*A.-

??r<   )>collectionsr  r  r   typingr   r   r   r   r   r   r	   r
   r   r   r   r   rI   torch.distributed.rpcdistributedrL   r   r   r   r   torch.distributedr   torch.distributed.nn.jitr   torch.distributed.rpc.internalr   torch.nnr   torch.nn.parameterr   torch.utils.hooksr   __all__r   r   1instantiate_non_scriptable_remote_module_templatero   r   
namedtupler!   r   r;   rG   rN   rM   rW   r   r[   r]   r   r  r  r#  r*  _register_reducerrJ   RecursiveScriptModulerh   r<   r:   <module>r:     sh    	 
      # # + + , 1 @  ( - 

fck"F*
+ Cx  CLBBD %% ! 1+001JLmn 1 -42$sxx	/B*C Js Jt J_BII _DSB= SBl,0
@ (  ' '6L M '  ' '	II##%Er<   