
    sgu$                     f   d dl Z d dlmZmZmZmZmZmZmZm	Z	m
Z
 d dlZd dlmZ d dlmZ d dlmZ ddlmZmZ dZ G d d	e      Zd
ededefdZ G d d      Z ee      	 ddej0                  deeej                  j0                        dej0                  fd       Zdej0                  defdZy)    N)	AnycastDictIterableListNoReturnOptionalSetTuple)_State)DistributedDataParallel   )_get_registrycontract c                       e Zd Zd fdZefdej                  deej                     deej                     de	ddf
dZ
dd	Zdej                  deej                     ddfd
ZddZddZdej                  deedf   dee	ef   defdZdej                  deej(                     dej(                  dej(                  fdZ xZS )_ReplicateStatereturnNc                     t         |           t        j                         | _        d| _        t        j                         | _        | j                  | _        g | _        d| _	        d | _
        i | _        g | _        y )NF)super__init__nnParameterListmodulehas_initialized_param_list_orig_module_param_names_no_sync
_init_args_init_kwargs_comm_hook_args)self	__class__s    Z/var/www/html/venv/lib/python3.12/site-packages/torch/distributed/_composable/replicate.pyr   z_ReplicateState.__init__   sk    !#!1!1!3%*-/-=-=-? !KK')#59,.*,    r   ignored_modulesignored_paramsprefixc                 b   t        |      ry ||v ry |t        k7  r| dnt        }|j                  d      D ]D  \  }}||vs| j                  j	                  |       | j
                  j	                  | |        F |j                         D ]  \  }}	| j                  |	||| |          y )N.F)recurse)r)   )_is_fully_sharded_ROOT_MODULE_PREFIXnamed_parametersr   appendr   named_children_collect_params)
r#   r   r'   r(   r)   recurse_prefixnpnamechild_modules
             r%   r2   z_ReplicateState._collect_params    s     V$ _$ #&99vhaL?R 	 ++E+: 	ADAq&  ''*!!((N+;A3)?@	A
 #)"7"7"9 	D,  ()$0	 ! 	r&   c                 P     t        j                  d       fd       } |        y )NT)	recursivec                       j                   J   j                   j                   i  j                    j                          d _         i  _        y )N )r    initr!   register_comm_hook)r#   s   r%   
_lazy_initz-_ReplicateState.lazy_init.<locals>._lazy_initA   sK    ??...DIIt<$*;*;<##% DO "Dr&   )torch_disable_dynamo)r#   r>   s   ` r%   	lazy_initz_ReplicateState.lazy_init@   s'    				.	# 
/	# 	r&   c                    | j                   ry d| _         |j                  dd       }|| _        |D ch c]  }|j                         D ]  }|  }}}|j	                         D ]-  }t        |      s|j                  |j                                / ddlm}	  |	||       | j                  |||       d|v rV|d   ;|d   }
t        |
t        j                        r|
j                  dk(  rd |d<   n|
g|d<   nd |d<   |j                  d       t        | j                   fi || _        t%        j&                  | j"                        t(        j+                  | j                        _        y c c}}w )	NTdevice_meshr   )_localize_dtensor)r(   	device_idcpu
device_ids)r   getr   
parametersmodulesr-   update%torch.distributed.tensor.parallel.ddprD   r2   
isinstancer?   devicetypepopr   r   _ddpweakrefref	replicatestate_ddp_weakref)r#   r   r'   kwargsrC   mr5   r(   	submodulerD   rE   s              r%   r<   z_ReplicateState.initK   sP    #jj5%4MallnM!M!MM) 	>I +%%i&:&:&<=	> 	L&@V_nE& 
 k".";/	i69>>U;R+/F<( -6;F<('+|$JJ{#+D,<,<GG	4;KK		4J	$1; Ns   E.c                     | j                   D ]!  \  }} | j                  j                  |i | # | j                   j                          y N)r"   rQ   r=   clear)r#   	comm_argscomm_kwargss      r%   r=   z"_ReplicateState.register_comm_hookw   sI    &*&:&: 	D"I{(DII(()C{C	D""$r&   c                      || _         || _        y r[   )r    r!   r#   argsrW   s      r%   record_init_argsz _ReplicateState.record_init_args|   s    "r&   ra   .rW   c                     | j                   s| j                  r| j                          | j                   | j                  _         | j                  j                  |i |S r[   )r    r!   rA   r   rQ   require_backward_grad_sync_pre_forward)r#   r   ra   rW   s       r%   forward_pre_hookz _ReplicateState.forward_pre_hook   sL     ??d//NN37==/@		,%tyy%%t6v66r&   inputoutputc                 8    | j                   j                  |      S r[   )rQ   _post_forward)r#   r   rg   rh   s       r%   forward_post_hookz!_ReplicateState.forward_post_hook   s     yy&&v..r&   r   N)__name__
__module____qualname__r   r.   r   Moduler
   	Parameterstrr2   rA   r<   r=   rb   r   r   r   rf   r?   Tensorrk   __classcell__)r$   s   @r%   r   r      s   -& *		 RYY BLL)	
  
@	*K		*K RYY*K
 
*KX%
#7ii7',S#X7@DS#X7	7/		/ U\\"/ 	/
 
/r&   r   ra   rW   r   c                      t        d      )NzGDDP does not support deepcopy. Please use state dict for serialization.)AssertionError)ra   rW   s     r%   unimplemented_deepcopyrw      s    
Q r&   c                   *    e Zd Zd ZdeddfdZddZy)DDPc                 L    | j                   d   } |j                  |g|i |S )z
        Override ``__new__`` to remove the DDP class and directly construct
        the original class for cases like indexing into a container module.
           )__mro____new__)clsra   rW   orig_clss       r%   r}   zDDP.__new__   s.     ;;q>x:4:6::r&   requires_gradient_syncr   Nc                 :    | t         j                  |       _        y)a  
        Sets if the module should sync gradients. This can be used to implement
        gradient accumulation without communication.

        Args:
            requires_gradient_sync (bool): Whether to reduce gradients for the
                module's parameters.
        N)rT   rU   r   )r#   r   s     r%   set_requires_gradient_synczDDP.set_requires_gradient_sync   s     .D)C	&r&   c                 d    t         j                  |       j                  j                  ||f       y r[   )rT   rU   r"   r0   r`   s      r%   r=   zDDP.register_comm_hook   s#    --44dF^Dr&   rl   )rm   rn   ro   r}   boolr   r=   r;   r&   r%   ry   ry      s#    ;	D 	D$ 	DEr&   ry   )	state_clsr   r'   c                    t         j                  j                  d       d|v r=t        |d   t        t         j
                  f      st        dt        |d                t        |       rt        d      |i }nt        |      }t        t        t        j                  |             }| j                  |j                  d       |j!                  dd      }|Fd	d
lm} |j'                  |      }||k7  r*d	dlm}m} | j                  |       | j/                  |       | j/                  |j0                          |j2                  | |fi | | j4                  }	dt6        i}
t        d|	j8                   t:        |	f|
      }|| _        | S )zReplicates a module

    Args:
        module (torch.nn.Module): module to replicate

    Example::
        >>> # xdoctest: +REQUIRES(module:torch._C._distributed_c10d)
        >>> module = nn.Linear(3, 3)
        >>> replicate(module)
    ztorch.distributed.replicaterE   z6Expected device_id to be int or torch.device, but got zGCannot apply `replicate()` on a Module already managed by `fully_shard`NT)with_kwargsrC   r   )_mesh_resources)rD   _reconstruct_dtensor__deepcopy__ry   )r?   _C_log_api_usage_oncerM   intrN   RuntimeErrorrO   r-   setr   r   rT   rU   register_forward_pre_hookrf   rH   torch.distributed.device_meshr   get_root_meshrL   rD   r   register_forward_hookrk   rb   r$   rw   rm   ry   )r   r'   rW   rU   rC   r   	root_meshrD   r   r~   dctnew_clss               r%   rT   rT      s     
HH  !>? f&-U\\/BC{ 3457 
  U
 	
 o.)//&"9:E
$$U%;%;$N**]D1KA#11+>	 #
 ,,-AB(():;
  !8!89E6?=f= 

C1
2CS'#sS9GFMr&   c                 &    t        |       }|yd|v S )z+Check if module is marked with fully_shard.Ffully_shard)r   )r   registrys     r%   r-   r-      s     V$HH$$r&   r[   )rR   typingr   r   r   r   r   r   r	   r
   r   r?   torch.nnr   #torch.distributed._composable_stater   torch.nn.parallelr   r   r   r.   r   rw   ry   rp   rT   r   r-   r;   r&   r%   <module>r      s     R R R   6 5 -  }/f }/@#   E E4 
O$ <@EIIEhuxx78E YY	E %EP%bii %D %r&   