
    sg              	       <   d dl mZmZmZmZmZ d dlmZ d dlm	Z	m
Z
 g Zdej                  defdZdeeej                  eej                  f      fdZdej                  d	efd
Zdddej                  dedeeej                        fdZdej                  fdZy)    )AnyListOptionalSetTupleN)_flatten_tensor_unflatten_tensormodulepathc                     d|v r;|j                  d      }dj                  |dd       }| j                  |      } |d   }| |fS )zH
    Get submodule and the direct path of parameter from the module
    .N)splitjoinget_submodule)r
   r   	path_listparent_module_paths       X/var/www/html/venv/lib/python3.12/site-packages/torch/distributed/tensor/parallel/ddp.py_get_submodule_n_paramsr      sQ     d{JJsO	 XXin5%%&89}4<    
param_listc                 l    | D ]/  }|\  }}}t        ||      sJ t        ||       t        |||       1 y)z-
    Update parameters within the module
    N)hasattrdelattrsetattr)r   itemparent_modulemodule_pathts        r   _update_module_paramr       sE      /(,%{A}k222{+{A.	/r   _inputc                     g }| j                         D ]H  \  }}t        |d      st        ||j                        }|j	                  g t        | |      |       J t        |       y)z:
    Recontruct DTensor parameters from local tensors
    _st_infoN)named_parametersr   r	   r#   appendr   r    )r
   r!   r   namer   dtensors         r   _reconstruct_dtensorr(   %   sl     J**, Qa1j!'1::6GO 7 EOwOPQ $r   )ignored_params_r)   c                
   |
t               }g }| j                         D ]W  \  }}||v rt        |      \  }}|t        j                  |      }||_        |j                  g t        | |      |       Y t        |       y)z5
    Convert DTensor parameters to local tensors
    N)	setr$   r   nn	Parameterr#   r%   r   r    )r
   r)   r*   r   r&   paramr   sharding_infos           r   _localize_dtensorr1   2   s     J..0 KeN"*51=$QA&AJI 7 EIqIJK $r   c                 r    t        | dd       | j                  t               | j                  t                y)a  
    Enable the composability between Tensor Parallelism (TP) and Data
    Parallelism(DP) in PyTorch when using DDP. We need to convert Parameters which
    are DTensors to local tensors before wrapping with data parallelism API.
    We then register two hooks, one for converting local tensors back to DTensor
    preforward and one to convert DTensors back to tensors after Forward. By
    integrating this way, we avoid any special handling of DTensor parameters by DDP
    and get DTensor's gradients propagated back to DP, e.g. gradient buckets of DDP.

    For now, this API only works with ``DistributedDataParallel``. It will later support
    other DP methods such as FSDP.

    Args:
        module (:class:`nn.Module`):
            Module which has been applied TP on.

    Example::
        >>> # xdoctest: +SKIP("distributed")
        >>> from torch.distributed.tensor.parallel import parallelize_module, PairwiseParallel
        >>> from torch.nn.parallel import DistributedDataParallel as DDP
        >>> from torch.distributed.tensor.parallel.ddp import pre_dp_module_transform
        >>>
        >>> # Define the module.
        >>> m = module(...)
        >>> parallelize_module(m, PairwiseParallel())
        >>> m = pre_dp_module_transform(m)
        >>> m = DDP(m)
        >>>
    N)r1   register_forward_pre_hookr(   register_forward_hook)r
   s    r   _pre_dp_module_transformr5   F   s.    > fdD)
$$%9:
  !23r   )typingr   r   r   r   r   torch.nnr-   6torch.distributed.tensor.parallel._data_parallel_utilsr   r	   __all__Modulestrr   r.   r    r(   r1   r5    r   r   <module>r=      s    2 2  	BII 	S 	/T%		30L*M%N /
% 
%C 
% OS%II%%08R\\9J0K%("4RYY "4r   