
    sgJ              	          d dl Z d dlZd dlmZ d dlmZ d dlmZ d dlm	Z	m
Z
mZmZmZ d dlZd dlmZ d dlmc mc mZ d dlmZmZmZ  e j4                  e      Z G d d      Zd	ej<                  j>                  d
ee e	e ee    f   f   fdZ!y)    N)defaultdict)contextmanager)Enum)DictIteratorListSetTuple)_apply_to_modules_get_module_fsdp_stateclean_tensor_namec                       e Zd ZU  G d dee      Z ee      Ze	eef   e
d<    e       Zee   e
d<   edd       Zeededed   fd	              Zed
eddfd       Zy)SimpleProfilerc                   $    e Zd ZdZdZdZdZdZdZy)SimpleProfiler.Typeall
all_gatherall_gather_object
reshardingH2DD2HN)	__name__
__module____qualname__ALL	ALLGATHERALLGATHER_OBJ
RESHARDINGr   r        V/var/www/html/venv/lib/python3.12/site-packages/torch/distributed/fsdp/_debug_utils.pyTyper      s      	+!
r    r"   results	profilingreturnNc                 l    | j                   j                          | j                  j                          y )N)r#   clearr$   )clss    r!   resetzSimpleProfiler.reset"   s"    r    profile_typec              #     K   || j                   vs
J | d       | j                   j                  |       t        j                         }	 d  t        j                         }| j                  |xx   ||z
  z  cc<   | j                   j                  |       y # t        j                         }| j                  |xx   ||z
  z  cc<   | j                   j                  |       w xY ww)Nzk is already being profiled. SimpleProfiler does not support profiling multiple instances at the same time. )r$   addtime	monotonicr#   remove)r(   r*   beginends       r!   profilezSimpleProfiler.profile'   s      3==0 	
n  	
0 	,' 	/.."CKK%u4%MM  . .."CKK%u4%MM  .s    AC'B A
C'AC$$C'msgc                     t        j                         dk(  rPt        j                         t         j                  j                  k(  r!t
        j                  d|| j                         | j                          y )Nr   z%s %s)	distget_rankget_debug_level
DebugLevelINFOloggerinfor#   r)   )r(   r3   s     r!   dump_and_resetzSimpleProfiler.dump_and_reset9   sI     ==?aD$8$8$:doo>R>R$RKKckk2		r    )r%   N)r   r   r   strr   r"   r   floatr#   r   __annotations__setr$   r	   classmethodr)   r   r   r2   r<   r   r    r!   r   r      s    sD  !,E 2GT#u*2%Is3x  /3 /8D> /  /     r    r   modelr%   c                     d }d }dg}i }t        | ||| j                         D cg c]  \  }}|	 c}}||      S c c}}w )a  
    It is used for composable fully_shard() code path, it returns
      1. sharded module tree info: each line reprents a submodule name that contats the
    submodule's FQN and its submodule class name, if the submodule is sharded by `fully_shard`,
    the submodule name will add a postfix with ' FULLY SHARDED'. Each increased tree
    level adds 4 spaces before the printed name. A printed sharded module tree info for a toy model
    is like this:
        [CompositeModel] FULLY SHARDED
            l1[Linear]
            u1[UnitModule] FULLY SHARDED
                u1.l1[Linear]
                u1.seq[Sequential]
                    u1.seq.0[ReLU]
                    u1.seq.1[Linear]
                    u1.seq.2[ReLU]
                u1.l2[Linear]
            u2[UnitModule] FULLY SHARDED
                u2.l1[Linear]
                u2.seq[Sequential]
                    u2.seq.0[ReLU]
                    u2.seq.1[Linear]
                    u2.seq.2[ReLU]
                u2.l2[Linear]
            l2[Linear]
      2. a dict mapping from the concated module FQN and class name to a list of its managed
    original parameters' FQNs. An example of the dict for the above toy sharded model is like this:
            {'[CompositeModel]': ['l1.weight', 'l1.bias', 'l2.weight', 'l2.bias'],
             'u1[UnitModule]': ['u1.l1.weight', 'u1.l1.bias', 'u1.seq.1.weight', 'u1.seq.1.bias', 'u1.l2.weight', 'u1.l2.bias'],
             'u2[UnitModule]': ['u2.l1.weight', 'u2.l1.bias', 'u2.seq.1.weight', 'u2.seq.1.bias', 'u2.l2.weight', 'u2.l2.bias']
            }
    All FQNs are prefixed starting from ``model``.

    Args:
        model (torch.nn.Module): Root module (which may or may not be passed to
                                 composable `fully_shard()`).
    c                 <   |dz  }t        |      dkD  r|d   dk(  r|d d n|}|dz   | j                  j                  z   dz   }d|z  |z   }t        |       }	|	|dxx   |dz   z  cc<   y |	j                  j                  | d       }
|
r|dxx   |d	z   dz   z  cc<   n|dxx   |dz   z  cc<   |
rl|
j                  }t        |t        j                        sJ |j                  D cg c]  }t        ||z          }}||v r||   j                  |       y |||<   y y c c}w )
N   r   .[] 
z FULLY SHARDED)len	__class__r   r   _fully_sharded_module_to_handleget
flat_param
isinstanceflat_param_fileFlatParameter_fqnsr   extend)moduleprefix
tree_levelsharded_tree_infosharded_module_name_to_fqns
num_spacestrimed_prefixprefixed_module_nameprinted_prefixed_module_namestatehandleparamnameglobal_fqnss                 r!   	module_fnzD_get_sharded_module_tree_with_module_name_to_fqns.<locals>.module_fnj   sa     !^
K!Or
c0AF3BK 	  -s2V5E5E5N5NNQTT'*Z'7:N'N$&v.=a $@4$GG 66::64Ha ,/??$F  a $@4$GG %%Ee_%B%BCCC=B[[59!&4-0K  $'BB+,@AHHUDO+,@A s   "Dc                     | d   |fS )Nr   r   )rY   rZ   s     r!   	return_fnzD_get_sharded_module_tree_with_module_name_to_fqns.<locals>.return_fn   s     #%@@@r     )r   named_parameters)rB   rd   rf   rY   rZ   key_s          r!   1_get_sharded_module_tree_with_module_name_to_fqnsrk   B   sa    P"PHA
 	$ 9; 1134a4#  	5s   =)"loggingr-   collectionsr   
contextlibr   enumr   typingr   r   r   r	   r
   torchtorch.distributeddistributedr5   "torch.distributed.fsdp._flat_paramfsdp_flat_paramrR   $torch.distributed.fsdp._common_utilsr   r   r   	getLoggerr   r:   r   nnModuler=   rk   r   r    r!   <module>r{      s      # %  3 3    < <  
		8	$) )X[88??[
3S$s)^$$%[r    