
    sgi                        d dl mZmZmZmZ d dlmZ d dlZd dlm	Z
 d dlmZ d dlmZ d dlmZmZ d dlmZ d dlmZ d dlmZmZmZmZmZmZmZmZmZm Z  d d	l!m"Z"m#Z#m$Z$ d d
l%m&Z& d dl'm(Z( d dl)m*Z*m+Z+m,Z,m-Z- d dl.m/Z/  ee       ede0      dddddddddddddejb                  dee
jd                     dee/   dee-   dee,   dee+   deeej                  jb                        deee3ejh                  f      deeejb                  gdf      de5de5deeeej                  jl                        eeej                  jb                        f   dejb                  fd               Z7y)!    )CallableIterableOptionalUnion)
deprecatedN)contract)_get_module_state_insert_module_state)
_FSDPState)_annotate_modules_for_dynamo)
_init_buffer_state_init_core_state_init_device_handle_init_ignored_module_states_init_param_handle_from_module_init_prefetching_state_init_process_group_state_init_runtime_state_init_state_dict_stateHYBRID_SHARDING_STRATEGIES)_register_post_forward_hook_register_pre_forward_hook_register_root_pre_forward_hook)_register_all_state_dict_hooks)
_auto_wrap)BackwardPrefetch
CPUOffloadMixedPrecisionShardingStrategy)_Policy)	state_clsa1  `torch.distributed._composable.fully_shard` is being deprecated. You can continue to use the wrapper based FSDP. See usage in: https://github.com/pytorch/pytorch/blob/main/torch/distributed/fsdp/fully_sharded_data_parallel.py. `torch.distributed._composable.fully_shard` will be removed after PyTorch 2.5.)categoryF)process_grouppolicystrategymixed_precisioncpu_offloadignored_modules	device_idparam_init_fnsync_module_statesforward_prefetchignored_statesmoduler#   r$   r%   r&   r'   r(   r)   r*   r+   r,   r-   returnc                   t         j                  j                  d       |t        |t              st        d|       t        j                  |       }t        || ||      }t        || |j                  |      }t        | |j                  d       t        ||||      }|X||||||||	|
|d
}|t        v r|j                  |j                   f|d<   t#        | ||j                  |j                  |t               t%        ||xs t&        j(                  ||dddd      }t+        |      }t-        |t.        j0                  |
      }t3        ||       }t5        || |||	      }t7        |      }t9        |       t;        ||        t=        ||        t?        ||        tA        | |       | jC                         D ])  }||jD                  v stG        |      tA        ||       + | S )	zDApplies ``FullyShardedDataParallel`` (FSDP) semantics to ``module``.ztorch.distributed.fully_shardzExpects a `_Policy` but got T)
r#   r%   r&   r'   r(   r)   r*   r+   r,   r-   r#      )limit_all_gathersuse_orig_paramsbackward_prefetch_limitforward_prefetch_limit)r,   )$torch_C_log_api_usage_once
isinstancer    
ValueErrorfully_shardstater   r   _ignored_paramsr   _ignored_modulesr   r   r#   _inter_node_pgr   r   r   
FULL_SHARDr   r   r   BACKWARD_PREr   r   r   r   r   r   r   r
   modules_fully_sharded_module_to_handler	   )r.   r#   r$   r%   r&   r'   r(   r)   r*   r+   r,   r-   r<   root_kwargs	submodules                  \/var/www/html/venv/lib/python3.12/site-packages/torch/distributed/_composable/fully_shard.pyr;   r;   (   s   4 
HH  !@A*VW"=7x@AAf%E'vWEvu/D/DiPE )?)?F%e]HfME* .&."*"4 0,
 11,1,?,?AUAU+VK(""!!	
 /$// ! 	E  &E#,,?OE uf-E*vy-1CE #5)E"5)uf-v.#E62 '^^% 3	>>>!),4 E23 M    )8typingr   r   r   r   typing_extensionsr   r6   torch.distributeddistributeddisttorch.nnnn&torch.distributed._composable.contractr   #torch.distributed._composable_stater	   r
   $torch.distributed.fsdp._common_utilsr   $torch.distributed.fsdp._dynamo_utilsr   "torch.distributed.fsdp._init_utilsr   r   r   r   r   r   r   r   r   r   %torch.distributed.fsdp._runtime_utilsr   r   r   (torch.distributed.fsdp._state_dict_utilsr   "torch.distributed.fsdp._wrap_utilsr   torch.distributed.fsdp.apir   r   r   r   torch.distributed.fsdp.wrapr    FutureWarningModuleProcessGroupintdevicebool	Parameterr;    rG   rF   <module>ra      s   6 6 (     ; W ; M   
 T 9  0 
JU  26 $+/04(,;?48;?$" 	SIIS D--.S W	S
 '(S n-S *%S huxx78S c5<</01S Hbii[$%678S S S %((,,-.%((//9R0SSS  YY!S  SrG   