sg ddlZddlZddlZddlZddlZddlmZmZddlmZddl m Z ddl m Z m Z ddlmZddlmZmZmZmZmZmZmZmZmZddlmZddlZddlmZddl m!Z!ddl"m!cm#Z$dd l%m&Z&dd l'm(Z(dd l)m*Z*m+Z+dd l,m-Z-dd l.m/Z/m0Z1ddl2m3Z3ddl4m5Z5ddl6m7Z7m8Z8m9Z9ddl:m;Z;ddlZ>m?Z?ddl@mAZAmBZBmCZCddlDmEZEmFZFmGZGmHZHddl mIZImJZJddlKmLZMddlNmOZOmPZPmQZQmRZRddlSmTZTmUZUddlVmWZWGdde ZXGdde ZYGdde!jeZ[d e!jd!ejd"efd#Z] djd e!jd$e^fd%Z_dkd&Z`d'Zad(Zbdld e!jd)e^fd*Zcd e!jd+e^fd,Zdd e!jd-e^fd.ZeGd/d0ZfGd1d2e[ZgGd3d4e[ZhGd5d6ehZiGd7d8ehZjGd9d:e[ZkGd;de!jZmGd?d@ehZnGdAdBe!jZoGdCdDe!jZqGdEdFe!jZrejdGefdHZtejdIefdJZuejdKefdLZveejdMefdNZweejdOefdPZxeejdQefdRZyeejdSefdTZzdUed"edVedWefdXZ{ dmdYe!jdZe!jd[ee|d\ffd]Z}Gd^d_ePZ~Gd`daeOZdndbeefdcZGdddee!jZGdfdge!jZGdhdie!jZy)oN)ABCabstractmethod) nullcontext)deepcopy)autoEnumwraps) AnyCallableDictList no_type_checkOptionalTupleTypeUnion)mock) checkpoint) fully_shard)FSDPParamGroupRegisterPostBackwardFunction) DeviceMesh) CPUOffloadFullyShardedDataParallel) TrainingState)#NO_RESHARD_AFTER_FORWARD_STRATEGIES)BackwardPrefetchMixedPrecisionShardingStrategy)ShardedGradScaler)always_wrap_policyModuleWrapPolicywrap)distribute_tensorDTensorShard)ColwiseParallelparallelize_moduleRowwiseParallelSequenceParallel)TransformerDecoderLayerTransformerEncoderLayer)DistributedDataParallel)MultiProcessTestCaseMultiThreadedTestCase run_subtests TEST_SKIPS) FILE_SCHEMAget_cycles_per_ms) has_tritonc(eZdZeZeZy) FSDPInitModeN)__name__ __module__ __qualname__rNO_FSDP RECURSIVEV/var/www/html/venv/lib/python3.12/site-packages/torch/testing/_internal/common_fsdp.pyr7r7CsfGIr>r7c6eZdZeZeZeZy) CUDAInitModeN)r8r9r:r CUDA_BEFORE CUDA_AFTER CUDA_NEVERr=r>r?rArALs&KJJr>rAceZdZdZedeejdffdZedejfdZ ed dZ e ede d e de jfd Zy) FSDPTestModelzZThis defines the interface expected from all models used commonly for FSDP unit tests.return.cy)z+Returns an input for the model as as tuple.Nr=selfdevices r? get_inputzFSDPTestModel.get_inputY r>cy)z,Returns the loss given the input and output.Nr=)rJinputoutputs r?get_losszFSDPTestModel.get_loss^rMr>Ncy)zargskwargscy)z&Initializes an instance of this model.Nr=)rVrWs r?initzFSDPTestModel.iniths r>rGN)r8r9r:__doc__rrtorchTensorrLrQrU staticmethodr nnModulerYr=r>r?rFrFUs 5s):#;        C 3 299  r>rFmodel process_group assert_fnc,|jDcgc]%\}}||jjf'}}}||jDcgc]%\}}||jjf'c}}z }t j |}t |D cgc]} d} } t j| ||| d} | J| ddD])} | Jt| | D]\\} } \} }|| |+ycc}}wcc}}wcc} w)a All-gathers module states across ranks and calls ``assert_fn`` on each pair of corresponding states from rank 0 and a nonzero rank. For example, if ``assert_fn`` is ``self.assertEqual()``, then this checks that all module states are equal across ranks. Ngroupr) named_parametersdetachcpu named_buffersdistget_world_sizerangeall_gather_objectzip)rarbrc param_nameparamnamed_module_states buffer_namebuffer world_size_olist rank0_statesstatep1p2s r?_assert_module_statesr}os6"'!7!7!9 J U\\^'')*#(#6#6#8 K fmmo))+,$$]3J , -aT -E -5"5]K8L  ## #qr   #L% 8  GQWa b"   .s*D*D ' D zero_buffersc|rtj|n t}|5|jD]/}t j 5|j ddd1|rB|jD]/}t j 5|j ddd1dddy#1swYxYw#1swYPxYw#1swYyxYw)zBZeros the parameters and optionally buffers of ``model`` in place.N)FSDPsummon_full_paramsr parametersr\no_gradzero_buffers)rar~ summon_fullctxrrrus r? _zero_modelrs -8$ ! !% ([]C #%%' E      --/ #]]_#LLN## # ##  ## ##s;(C  B43C C! C 4B= 9C C C  Ccj|s|j}|r|j|jSN)cudahalf state_dict)ra cpu_offloadrs r?_get_state_dictrs+       r>c jdj|Dcgc]}||t|ndc}Scc}w)Nrwnone)joinstr)test_name_mappingrVss r? subtest_namers7 88IMNAam 3q6 " ?N Ns0c@|jD];\}}|jtjdk7s)|j||<=|dk(r|ndg}t j ||d}|j D]}||j||<|S)Nrjr)itemsrKr\rjrlbroadcast_object_listkeysr)rankrrqrrrxs r?_broadcast_state_dictrs(--/1 E <<5<<. .%*YY[Jz "1 19Z$ /Eu%qJ oo'? !+J!7! :? r>recursectj||5tt|j cdddS#1swYyxYw)a[ Returns the full unsharded parameters of ``model``. Any FSDP-managed parameters offloaded to CPU are moved to GPU in the returned list. Args: recurse (bool): If ``False``, only unshards the parameters immediate to ``model``; if ``True``, recurses through the module hierarchy rooted at ``model``. )rN)rrrlistr)rars r?get_full_paramsrs?   82U--/01222s "AA  move_to_cudac*|r|jS|Sr)r)rars r? _maybe_cudars'5::<2U2r> wrap_fsdpc(|s|St|g|i|Srr)rarrVrWs r?_maybe_wrap_fsdprs!5CtE'CD'CF'CCr>c:eZdZdedefdZdefdZdefdZdZy) DummyProcessGrouprsizec ||_||_yr)_rank_size)rJrrs r?__init__zDummyProcessGroup.__init__s  r>rGc|jSr)rrJs r?rzDummyProcessGroup.rank zzr>c|jSr)rrs r?rzDummyProcessGroup.sizerr>cBtj}d}||_|S)Ncdtjj}|jd|S)Nrg)r\futuresFuture set_result)futures r? get_futurez/DummyProcessGroup.allreduce..get_futures'+0==+?+?+AF   a Mr>)rMockr)rJrVrW dist_waitrs r? allreducezDummyProcessGroup.allreduces"IIK   * r>N)r8r9r:intrrrrr=r>r?rrs2Scc r>rceZdZdejdededeffd ZdZdZ dZ d Z e ddejd e ded eeeefdeded eej(effd ZdZxZS)TransformerWithSharedParamsrfcuda_init_modeadd_bn deterministicct||j|_|j|_|rt j dd}d}tj|||_ tj|dddd|_ tj|||_ |jj|j_|jd|jjj!|f|jd t j"|j$t j& d|_|r)t jj+|j(nt jj-|_|t0j2k(r|j5}|r|j7yy) Nrg?)d_modelnum_encoder_layersnum_decoder_layersdim_feedforwarddropout vocab_bias long_buffer)dtype)superrrrrvr\ manual_seedr_ Embedding embed_tokens Transformer transformerLinear output_projweightregister_buffernew_ones zeros_likerlongbs BatchNorm1dIdentitybnrArBreval)rJrfrrrd_vocabr __class__s r?rz$TransformerWithSharedParams.__init__s^ JJL **,    a LL':>>    99Wg6#'"3"3":":  $++22;;WJG      T__EJJ ? 39%((&&tww/uxx?P?P?R \55 599;D  IIK r>c,tjd|jztjd|j d|j }tj|j dz|j d|j }||fS)Nrg rK)r\rrarangeviewr)rJrKsrctgts r?rLz%TransformerWithSharedParams.get_inputsl !dii-(ll2f-221dgg>ll477Q;v6;;AtwwGSzr>c|j|}||jz|jj|z}|j|}|j |}|j ||}|j |Sr)rrrtype_asrrr)rJsrc_idstgt_idsrrxs r?forwardz#TransformerWithSharedParams.forwardsv(DOO#d&6&6&>&>s&CC(ggcl   S# &""r>c|\}}tjj|jd|j d|jddS)Nsum) reduction)r_ functional cross_entropyrr)rJrOrPrwrs r?rQz$TransformerWithSharedParams.get_loss%sI3}}** KKFKKO ,chhrle+  r>c$|jyrbackwardrSs r?rUz(TransformerWithSharedParams.run_backward+  r>fsdp_init_mode fsdp_kwargsrGcD|i}|tjk(r&t|tr|d}n|}t ||||S|tj k(rd|vrt tth}n|jd}d|vr8|dtjtjhvrt|tsd}n|}t|tr|d} n|} t | |||} t| |fd|i|} |tjk(r| j!} | St#d|)ao Initializes a :class:`TransformerWithSharedParams` instance. Args: fsdp_init_mode (FSDPInitMode): If ``NO_FSDP``, then does not wrap any modules with FSDP. If ``RECURSIVE``, then wraps with top-level FSDP. By default, the top-level FSDP uses the ``ModuleWrapPolicy`` for encoder and decoder layers, but a different auto wrap policy may be specified via ``fsdp_kwargs``. cuda_init_mode (CUDAInitMode): Determines model movement to CUDA. fsdp_kwargs (Optional[Dict[str, Any]]): Optional keyword arguments forwarded to the FSDP constructor. deterministic (bool): Whether to make the model deterministic across constructions. add_bn (bool): Whether to include batch norm in the model. Nrauto_wrap_policysharding_strategyUnsupported FSDP init mode: )r7r; isinstancetuplerr<r#r-r,popr HYBRID_SHARD_HYBRID_SHARD_ZERO2rrArCr ValueError) rfrrrrrpgrfsdp_pg tformer_pgm fsdp_models r?rYz TransformerWithSharedParams.init.sT6  K \11 1%'1X.NFM |55 5!4#3//$ $/??3E#F ${2 34$113C3W3WXY"5%0%'"1X " +NFMA"2 J !8!88'__.  77GHIIr>c|jgSr)rrs r?get_ignored_modulesz/TransformerWithSharedParams.get_ignored_modules|s  !!r>)NFT)r8r9r:rl ProcessGrouprAboolrrLrrQrUr^r7rr rr rr_r`rrYr __classcell__rs@r?rrs(  (%( (  (T #  15# KJ  KJ$KJ%KJd38n- KJ  KJ  KJ ryy$ KJKJZ"r>rceZdZdejdededeffd ZdZdZ dZ d Z e ddejd e ded eeeefded ej&f d ZxZS)NestedWrappedModulerfrrrc t|j|_j|_|t j k(}fd}|rtjdtjttjdd||tj|ttjdd|ttjdd||ttjdd|ttjdd||_ y)Nc&r t|fiS|Srrlayerrrfrs r? _maybe_wrapz1NestedWrappedModule.__init__.._maybe_wrapE58K88Lr>rrrr)rrrrrvrArBr\rr_ Sequentialrrmodule rJrfrrrrrrrs `` ` r?rzNestedWrappedModule.__init__s JJL **,%)A)AA      a mm  !Q 6   BIIa,cxtjd|jztjdd|fS)Nrgrrr)r\rrrandrIs r?rLzNestedWrappedModule.get_inputs. !dii-( 1a/11r>c$|j|SrrrJrs r?rzNestedWrappedModule.forward{{1~r>c&|j}|Sr)rrJrOrPrTs r?rQzNestedWrappedModule.get_lossszz| r>c$|jyrrrSs r?rUz NestedWrappedModule.run_backwardrr>rrrGc|i}|tjk(rt|d||S|tjk(r5t|fd||d|}|tj k(r|j }|Std|)a Initializes a :class:`NestedWrappedModule` instance. Args: fsdp_init_mode (FSDPInitMode): If ``NO_FSDP``, then does not wrap any modules with FSDP. If ``RECURSIVE``, then wraps some nested modules with FSDP but not the top-level module. The model may later be wrapped with a top-level FSDP external to this method if desired. cuda_init_mode (CUDAInitMode): Determines model movement to CUDA. fsdp_kwargs (Optional[Dict[str, Any]]): Optional keyword arguments forwarded to the FSDP constructor. deterministic (bool): Whether to make the model deterministic across constructions. FrrrTr)r7r;rr<rArCrr)rfrrrrr s r?rYzNestedWrappedModule.inits.  K \11 1&-+   |55 5,-+   J!8!88'__.  77GHIIr>NF)r8r9r:rlrrrArrLrrQrUr^r7rr rr r_r`rYrrs@r?rrs     %    @2 15# +J  +J$+J%+Jd38n- +J  +J  +J+Jr>rc heZdZe ddej dededee e e fde f fd Z xZS)AlwaysWrapNestedWrappedModulerfrrrrc&ttt |tj|||}|tjk(r|S|tj k(r=|xsi}t |fdti|}|tjk(r|j}|Sy)z Initializes a :class:`NestedWrappedModule` instance, but unlike :meth:`NestedWrappedModule.init`, for the ``RECURSIVE`` init mode, this wraps with top-level FSDP and the ``always_wrap_policy()`` auto wrap policy. )rfrrrrrN) rr,rYr7r;r<rr"rArCr)rfrrrrrar rs r?rYz"AlwaysWrapNestedWrappedModule.inits )+H '//)#'    \11 1L |55 5%+KeX6HXKXJ!8!88'__.   6r>r*)r8r9r:r^rlrr7rArr rr rrYrrs@r?r,r,s^ 15#   $%d38n-   r>r,c eZdZdejdededeffd Zed dZ e d dejde dede e e efdef d ZxZS) NonUniformReqGradNWMrfrrrc tt| j|_j |_|t jk(}fd}|rtjdtjttjdd||tj|ttjdd|ttjdd||tjttjdd|ttjdd||_y)Nc&r t|fiS|Srrrs r?rz2NonUniformReqGradNWM.__init__.._maybe_wraprr>rrrr)rrrrrrvrArBr\rr_rrrrrs `` ` r?rzNonUniformReqGradNWM.__init__s !413JJL **,%)A)AA      a mm  !Q 6   BIIa,   r>c|jD]-\}}tj||r|jd/yr*)rhrematchrequires_grad_)ra req_grad_masknps r?_set_nonuniform_req_gradz-NonUniformReqGradNWM._set_nonuniform_req_grad+s:**, (DAq88M1-  ' (r>rrctjd}|tjk(r't |d||}tj |||S|tj k(rO|i}t |fd||d|}|tjk(r|j}tj |||Std|)a Initializes a :class:`NestedWrappedModule` instance, but unlike :meth:`NestedWrappedModule.init`, it wraps a second :class:`torch.nn.Sequential` container to enable the desired non-uniform ``requires_grad`` ``use_orig_params=True`` tests. For both ``RECURSIVE`` and ``NO_FSDP`` init modes, freezes all parameters except the last two to validate ``ShardedGradScaler`` support for ranks with no (non-zero sized) local shards in FSDP ``use_orig_params=True`` mode. zmodule\.2.*\.1.*Fr)Tr) r3compiler7r;r/r9r<rArCrr)rfrrrrreq_grad_pattern ddp_modelr s r?rYzNonUniformReqGradNWM.init1s(::&9: \11 1,-+ I ! 9 9)EU V  |55 5" --+   J!8!88'__. 9 9*FV W 77GHIIr>rZr*)r8r9r:rlrrrArr^r9r7rr rr rYrrs@r?r/r/s(   ( ( % (  ( T((  15# +J  +J$+J%+Jd38n- +J  +J+Jr>r/c eZdZdZdej dedeffd ZdZdZ dZ d Z e d e ed ededed ef d ZxZS)ModuleWithDelayzThis class wraps a :class:`FSDPTestModel` to optionally add a delay after computing the loss and/or before the gradient reduction.rdelay_after_loss_msdelay_before_reduction_mscLt|||_||_||_yr)rrr@rAr)rJrr@rArs r?rzModuleWithDelay.__init__ds' #6 )B& r>c8|jj|Sr)rrLrIs r?rLzModuleWithDelay.get_inputos{{$$V,,r>c$|j|Srr"r#s r?rzModuleWithDelay.forwardrr$r>c|jj||}|jdkDr=tjj t |jtz|SNr)rrQr@r\r_sleeprr4r&s r?rQzModuleWithDelay.get_lossusQ{{##E62  # #a ' JJ  c$":":=N=P"PQ R r>ctjjfd}tjd|5j j |dddy#1swYyxYw)NcjdkDr=tjjt jt z|i|SrF)rAr\rrGrr4)rVrWorig_reduce_scatterrJs r?_delayed_reduce_scatterz=ModuleWithDelay.run_backward.._delayed_reduce_scatter~sL--1 !!669J9LLM'77 7r>z'torch.distributed.reduce_scatter_tensor)r\ distributedreduce_scatter_tensorrpatchrrU)rJrTrKrJs` @r?rUzModuleWithDelay.run_backward{sW#//EE 8ZZ 57N  + KK $ $T * + + +s AA' module_class model_args model_kwargsc<t|j|i|||S)aA Args: module_class (Type[FSDPTestModel]): Wrapped module class to which to add delays. model_args: Positional arguments forwarded to the ``module_class`` ``init()``. delay_after_loss_ms (int): Delay after computing the loss/before the optimizer step (in ms). delay_before_reduction_ms (int): Delay before reduce-scattering gradients (in ms). model_kwargs: Keyword arguments forwarded to the ``module_class`` ``init()``. )r?rY)rOr@rArPrQs r?rYzModuleWithDelay.inits,* L  z :\ :  %  r>)r8r9r:r[r_r`rrrLrrQrUr^rrFr rYrrs@r?r?r?`sF   ! $' - + =)  ! $'     r>r?c~eZdZeej ddddfdejdedede e e e fde d ed efd Zy) NestedWrappedModuleWithDelayNFrrfrrrrr@rAc Dtjt|||||||S)Nrfrrrrr@rA)r?rYrrVs r?rYz!NestedWrappedModuleWithDelay.inits4## ))#' 3&?$  r>)r8r9r:r^rArCrlrr7rr rr rrrYr=r>r?rTrTs(4'>'>04##$)*    $ % d38n-    !  $'  r>rTc$eZdZfdZdZxZS)DummyDDPc0t|||_yr)rrr)rJrrs r?rzDummyDDP.__init__s  r>c&|j|i|Srr"rJrVrWs r?rzDummyDDP.forwardst{{D+F++r>r8r9r:rrrrs@r?rXrXs ,r>rXceZdZdejdedededef fd ZdZ dZ e d dejd e ded e eeefdedef d ZxZS) MixtureOfExpertsrfrrdelay_before_free_msrc t|||||||_||_||_|t j k(|_|r"tjd|jzd}d}d} ttj|||j} td| jD|_| jD] } d| _ |rtjdttj|||j} |rHtj$j'|jg} t)| | fi|} t)| |fi|} tj*ttj| ||j| | ttj|| |j|_y) N)rfrrr*rrrc3<K|]}|jywr)numel).0r8s r? z,MixtureOfExperts.__init__..s$L1QWWY$LTr)rrrfr_rrArBrr\rrrr_rrrnum_expert_paramsexpertrL new_grouprrr)rJrfrrr_rrd_expertd_sharedd_inputrhr8shared expert_grouprs r?rzMixtureOfExperts.__init__s )'   $8!"*l.F.FF    b499n -RYYx:D+>F&%7;7Fmm  '84d6G6G H    (G4d6G6G H   r>cfjdkDrjd}t|tretj j jjfd}tjd|5j|cdddSj|S#1swYxYw)Nrrctjjtjt z|i|Sr)r\rrGrr_r4)rVrW orig_reshardrJs r?_delayed_reshardz2MixtureOfExperts.forward.._delayed_reshards>JJ%%D558I8KKL(888r>z.torch.distributed.fsdp._runtime_utils._reshard) r_rrrr\rLfsdp_runtime_utils_reshardrrN)rJrrhrrrqs` @r?rzMixtureOfExperts.forwards  $ $q ([[^F&$'$0055DDMM 9ZZDFV* ;;q>** {{1~ **s ;B''B0c|j|jstj5|j D]v}t |dr|j |j j|jtjj|j |jx dddyy#1swYyxYw)Nrhre) rrr\rrhasattrgraddiv_rvrL all_reducerf)rJrTr8s r?rUzMixtureOfExperts.run_backwards ~~ O*OAq(+ vv) DOO4))44QVV4::4N O O O O Os-CACCrrc|i}|tjk(rt|d|||S|tjk(r6t|fd|||d|}|tj k(r|j }|Std|)a Initializes a :class:`MixtureOfExperts` instance. Args: fsdp_init_mode (FSDPInitMode): If ``NO_FSDP``, then does not wrap any modules with FSDP. If ``RECURSIVE``, then wraps some nested modules with FSDP, including the expert and shared layers, but not the top-level module. The model may later be wrapped with a top-level FSDP external to this method if desired. cuda_init_mode (CUDAInitMode): Determines model movement to CUDA. fsdp_kwargs (Optional[Dict[str, Any]]): Optional keyword arguments forwarded to the FSDP constructor. deterministic (bool): Whether to make the model deterministic across constructions. delay_before_free_ms (int): Delay before resharding expert parameters in the forward pass (in ms). F)rrr_rTr)r7r;r^r<rArCrr)rfrrrrr_r s r?rYzMixtureOfExperts.inits4  K \11 1#-%9+  |55 5)-%9+   J!8!88'__.  77GHIIr>)NFr)r8r9r:rlrrrArrrrUr^r7rr rr rYrrs@r?r^r^s2   2 2 % 2 " 2  2 h( O 15#$% 0J  0J$0J%0Jd38n- 0J  0J " 0J0Jr>r^c eZdZ ddddddedeej deded ef fd Zd ejd ejfd Z dZ xZ S)MLPTFr)bias with_bufferdim_multiplierdimrKr~rrc t|tj|||z|||_tj||z||||_|r)|j dtj|f|yd|_ y)N)rKr~rur) rrr_rin_projout_projrr\randnru)rJrrKr~rrrs r?rz MLP.__init__Pso yyns&:6PTU  .3"6FQUV   5;;vf+M NDKr>rrGc|j|}tj|}|j|}tj|}|j||jz}|Sr)rFrelurru)rJrzs r?rz MLP.forwardasS LLO FF1I MM!  FF1I ;; "DKKAr>c|j4tjjj |jyyr)rur\r_rYnormal_rs r?reset_parameterszMLP.reset_parametersjs+ ;; " HHMM ! !$++ . #r>r) r8r9r:rrr\rKrrr]rrrrs@r?r}r}Osv*. ! &    "%,,/r>r}cFeZdZdddedeffdZdededed dfd ZxZS) MLPStackF)with_seq_parallelmlp_dimrct|dt|t|dg}|r&|jtj|dt ||||_y)N)rFr~)r}appendr_ LayerNormrrr)rJrrmodulesrs r?rzMLPStack.__init__psX  * L  * $   NN2<<e< = '"!2r>tp_meshdp_meshuse_activation_checkpointingrGc tdtdtdtdtd|jrttdn td}|jrt d|d<t ||||D]8}t |tjr|r t|t|fd |i|:t|fd |i||S) NF)use_local_outputrg)output_layouts)z 0.in_projz 0.out_projz 1.in_projz 1.out_projz 2.in_projz 2.out_proj) sequence_dim3) device_meshparallelize_planmesh) r(r*rr'r+r)rr_rrr)rJrrrrrrs r? parallelizezMLPStack.parallelize|s)%@)5A(%@)5A(%@%%*qB "    ! !$4!$D S !4WGWX =F&",,/+6"  ) r8r9r:rrrrrrrs@r?rrosDBG 3 34 3'+   r>rceZdZdZddedeffd Zdejde e ejejfejffdZ xZ S) DoubleLinearz This can be used for returning multiple outputs from a module (``use_second_linear=True``) or for having an unused module (``False``). ruse_second_linearct|tj|||_tj|||_tj |_||_yr) rrr_rlin1lin2ReLUrr)rJrrrs r?rzDoubleLinear.__init__sG IIc3' IIc3' GGI !2r>rrGc|jr@|j|j||j|j|fS|j|j|Sr)rrrrr#s r?rzDoubleLinear.forwardsQ  ! !99TYYq\*DIIdiil,CC Cyy1&&r>T) r8r9r:r[rrrr\r]rrrrrs@r?rrsT 3C3D3'' uU\\5<</0%,,> ?'r>rnew_all_gather_into_tensorc#Ktj}tj|t_ dtj|t_y#tj|t_wxYwwr)rlall_gather_into_tensorbarrier)rorig_all_gathers r?patch_all_gatherrsO11OLLN"  0=$- 0=$-rrJrVrWct|dkDr|d}nd|vr|d}ntd|d|||||i|S)NrrPz,Cannot get reduce-scatter output from args: z kwargs: )lenAssertionError)clsrJrcrVrWrPs r?reduce_scatter_with_assertrsa 4y1}a V !;D6F8 T  f  / //r>replicated_modulesharded_moduleprefixes_to_ignore.ct|j|jD]\\}}\}}|}|D]} |j| d}|j|||j |t t |t sJ|j|j} } t| tdtdfk(r tdt|| | } |j|j| j|j|j|j |j!|jt|j| | } |j |jt t |jt sJ|j|jj| jy)NrzmFSDP's (Shard(0), Shard(0)) layout differs from distribute_tensor(), so we cannot check for equality using it)rprhreplace assertEqualassertIsInstancer&rr placementsrr'rr%to_localrx assertIsNoneassertIsNotNone)rrrrreplicated_namereplicated_param sharded_name sharded_paramclean_sharded_nameprefixrrsharded_ref_paramsharded_ref_grads r?check_sharded_parityr"s OR**,n.M.M.OOTJ+*-JlM*( HF!3!;!;FB!G  H );< ]G4-111(44m6N6Nj  q58 4 4 ; ..>jQ  ..02C2L2L2NO  (   ]// 0  M../,-=-B-BD*U ]//9-,,g666  **3357G7P7P7RS1Tr>c@eZdZedZfdZdZdZdZxZ S)FSDPTestMultiThreadc~tjjrtjjSdS)Nr)r\r is_available device_countrs r?rvzFSDPTestMultiThread.world_sizeDs),1JJ,C,C,Euzz&&(L1Lr>cBt||jyr)rsetUp_spawn_threadsrJrs r?rzFSDPTestMultiThread.setUpHs   r>c t|g|i|Srr1r[s r?r1z FSDPTestMultiThread.run_subtestsLD242622r>c@tjjyrr\_dynamoresetrs r?perThreadSetUpz"FSDPTestMultiThread.perThreadSetUpO r>c@tjjyrrrs r?perThreadTearDownz%FSDPTestMultiThread.perThreadTearDownRrr>) r8r9r:propertyrvrr1rrrrs@r?rrCs+ MM3r>rc$eZdZfdZedZedZedZdZdZ dZ dZ e d Z d%d ejd edededeededeedededeeeeffdZd dded d d d d d d d d f deedededeededededeed ee deed!ed"ededed#eeeefdeeeeff d$Z!xZ"S)&FSDPTestcht|dtjd<|j y)N0TORCH_NCCL_DESYNC_DEBUG)rrosenviron_spawn_processesrs r?rzFSDPTest.setUpWs)  14 ,- r>ctjjr(ttjj dSdS)Nrr)r\rrminrrs r?rvzFSDPTest.world_size_s149JJ4K4K4Ms5::**,a0TSTTr>c>tjjSr)rldistributed_c10d_get_default_grouprs r?rbzFSDPTest.process_groupcs$$7799r>c*t|jSr)r3 file_namers r? init_methodzFSDPTest.init_methodgst~~.//r>c<|j||jyr)rr)rJr rs r?_check_cpu_offloadzFSDPTest._check_cpu_offloadks j&<&<=r>c<|j||jyr)rbackward_prefetch)rJr rs r?_check_backward_prefetchz!FSDPTest._check_backward_prefetchns *J,H,HIr>c<|j||jyr)rforward_prefetch)rJr rs r?_check_forward_prefetchz FSDPTest._check_forward_prefetchqs ):+F+FGr>c t|g|i|Srrr[s r?r1zFSDPTest.run_subteststrr>c ||}||_||_|jdd}td|jd|jt j jrdnd} |r`t jjjjj} tjd|j|| n@tj|j|t!|j|j d} t j jrkt j j/rM|jt j j/z} t j j1| | g} tj2| t j4j7|j9||t j4j7tj2| tj:y#t"$r=} d | j$d vr&t'j(t*d j,d} ~ wwxYw)Nfake_pgFz dist init r=z, world=ncclgloofake)backendrvrstore)r rrvr recompilerbackend_unavailable) device_ids)rr getprintrvr\rrtesting _internalrLr FakeStorerlinit_process_groupr r RuntimeErrorrVsysexitr2 exit_coder set_devicerrrrun_testdestroy_process_group) rr test_namer piperWrJrrrer device_ids r?_runz FSDPTest._runws9~ "**Y. TYYKx/@AB"JJ335&6  //;;CCMMO''"# '' $ 0 0#"4??3  :: " " $)@)@)B EJJ$;$;$==I JJ ! !) ,#J +  i&  + ""$/ affQi'$9:DDE   s1B"H-- I368I..I3NFra num_stepsautocastlrfsdp_cpu_offload save_modelmixed_precisionenable_sharded_grad_scaler use_pure_fp16sharded_grad_scaler_kwargsc |xr |j} t|jj} | i} t d d|i| } t j j|j|d}t|D]}|jt jjd|5|jjt jd}| s|rMt|ts=t|t j r|j#}nt%d|D}||}| rft|trV|j&t(vrD|jD]1}|j+|jt jd3|jj-||j/| }ddd| j1}|s&| s$|j2t j4k(sJd| r+|j+|j2t j6net|tr+|J|j+|j2|j8n*|j+|j2t j4|jj;|| rTt|trD|jD]1}|j+|jt jd3| j=|| j?|s|jAjCDcic]\}}||jE}}}tG||jI|t|tr|jKtLjNjQS#1swYxYwcc}}w) Nenabledg?)r3momentumr)r;c3<K|]}|jywr)r)rdrs r?rez4FSDPTest._train_for_several_steps..s%>1affh%>rfrjzeloss data type should be float32, as the original parameter data type is float32.r=))offload_paramsnextrrKr!r\optimSGDrn zero_gradampr2rrLrrr]rrrrrrQtoscalerfloat32float16 param_dtyperUstepupdaterrclonerload_state_dict _assert_staterIDLEri)rJrar1r2r3r4r5r6r7r8r9cpu_offload_params model_devicesharded_grad_scalerr@rwrOrPr8rTkvrs r?_train_for_several_stepsz!FSDPTest._train_for_several_stepss4.Q2B2Q2QE,,./66 % -)+ &/ . 2L    0 0 2rCHy!9 2A OO ##FH#= M ..u||F/CD _Zt=T!%6 %  %%>%> >'"5$///>?#--/H((5<<3FGH||,,UF;>>|L- M.',,T2D"=JJ%--/555/!$$TZZ?t,*666$$TZZ1L1LM$$TZZ? LL % %d +!j&=))+DA$$QXXu||E/BCD $ $U +  & & (7<7G7G7I7O7O7QRtq!alR RE"%%j1s9 2v eT "    2 2 3{{}w M MfSs5DOO O rT model_classrr ref_init_fn num_itersrrrruse_orig_params init_kwargsc 8|tjk7sJd|i}d}|jj}|j|jtjt j fddi|}|t||g|}n||}|r|j}|j||| du||| | || }t|j}|j||| | | | d |j|j|||fddi|}t!|t"st#||jfi|}|r|j}|t j$k(r|j'}|duxr |j(}|xr|t j$k(}|xr|t j$k7}|rFt+j,d }|jD]}|j/|j,| |r|j1t2d n t5}|5|j||d |||| | || } ddd|ry|rVt+j,d }|jD]}|j/|j,|  j'} t7|}!t*j8j;| d | |s|j/||!ddyyy#t$r }td |d t||d}~wwxYw#1swYxYw)a Tests FSDP training against a reference, which defaults to DDP but may be customized with ``ref_init_fn``. Args: model_class (Type[FSDPTestModel]): A model class that inherits from ``FSDPTestModel``, which defines the expected interface. fsdp_init_mode (FSDPInitMode): The mode to initialize the FSDP-wrapped model. This should not be ``NO_FSDP``. ref_init_fn (Optional[Callable]): A callable to invoke that wraps a non-wrapped model to construct the reference model, where this wrapper should provide data parallel semantics. If ``None``, then the callable defaults to the DDP constructor. z.Expects an FSDP init mode that wraps with FSDPN{Gz?rT)r output_device)r2r3r4r6r7r8r9)rrrr6rrXz Initializing z raised error rjzSAn FSDP-managed module with parameter CPU offloading enabled has parameters on cudaF)r2r3r4r5r6r7r8r9) check_dtypezFSDP did not match DDP) exact_devicemsg)r7r;rbrrYrArBDDPrrTrrrJ ExceptionrrrrrCrr>r\rKrassertRaisesRegexr%rrr! assert_close)"rJrUrrrVrWr5rrrr6rrXr7r8rYr9rr3rra ref_modelref_loss ddp_paramsr r.r>expects_device_errorexpects_cpu_device cpu_devicerrcontext fsdp_lossfsdp_unsharded_paramss" r?_test_fsdp_parityzFSDPTest._test_fsdp_paritysF l22 2 < ; < 2  K !!&&(        $ $       EtfDII#E*I !(I00  $D0(+'A''A1  )..01 *%6%6#2$4#2    Y)))""  #   J*d+j$*<*<L LJ #*J \44 4#*J$D0O[5O5O  H~1H1HH   H~1H1HH  e,J#..0 ;  z: ;$  " ")    55!,% /+E++E6 I   e,J#..0 ;  z: ;!(I / ;  ""8YE"J  "=   %!,  ,9 "K Y}[MAxPQWX X YF  s$2"K$L$ L -LL L)r[NFNFFN)#r8r9r:rrrvrbr r rrr1 classmethodr0r_r`rrfloatrrrr rr rTrrFr7rAr rr rmrrs@r?rrVs* UU::00>JH33%3%t15 48+0#?CUyyUU U  U #:. UU".1U%)UU%-T#s(^$<Ux+/",,8<8<48!& %+0#04?C#b-(b%b% b h' b  bb b$$45b$$45b".1bbb%)bb d38n-!b"%-T#s(^$<#br>rcompile_compute_on_modulec@fdGddtfd}|S)Nctjjjj|i|t |dr|dj yyrF)r\rL _composablersrrr;)rVrWrps r?!fully_shard_with_compiled_computez=test_compiled_fsdp..fully_shard_with_compiled_computesT %%**66GG $ , G.1  GOO 1 r>c(eZdZeZeZy)*test_compiled_fsdp..FullyShardModeN)r8r9r:rEAGERCOMPILED_COMPUTEr=r>r?FullyShardModervs6r>ryc4tfd}|S)Nctjjjj}D]}|j k7r t stjd3tjjj}tjjj}tjj|j k(r|}n^|jk(rAdtjj_ dtjj_ }nt!d|| j"|j$< |i|tjj| j"|j$<|tjj_ |tjj_ y)Nz0Inductor on GPU needs Triton and recent GPU archTrgz!Need to implement FullyShardMode=)r\rLrsrsrrwr5warningswarnrconfigskip_fsdp_hooks _inductorcompile_threadsrrxNotImplementedError __globals__r8) rVrWoriginal_fully_shardmodeoriginal_skip_fsdp_hooksoriginal_compile_threadsfully_shard_patchryrtfuncs r?wrapperz6test_compiled_fsdp..decorator..wrappersj#(#4#4#@#@#E#E#Q#Q & R>/// MM"TU+0==+?+?+O+O(+0??+A+A+Q+Q(!!))+>///(<%^<<<;?EMM((8=>EOO**:(I%-;D6B CT  !5!>!>?d%f%!!))+BV  !5!>!>?7O $$49Q&&69 Rr>r )rrryrts` r? decoratorz%test_compiled_fsdp..decorators# t R  R@r>)r)rprryrts` @@r?test_compiled_fsdprs""""H r>c&eZdZdfd ZdZxZS) SkipModulec\t|tjddd|_yN Fr)rrr_rlinrs r?rzSkipModule.__init__s" 99R%0r>c$|j|Sr)rr#s r?rzSkipModule.forwardsxx{r>rZr\rs@r?rrs 1r>rc$eZdZfdZdZxZS) NestedLinearct||r5ttjdddj |_ytjdddj |_yr)rrr$r_rr nested_linear)rJ fsdp_wraprs r?rzNestedLinear.__init__sR  !%biiBU&C&H&H&J!KD !#2r!>!C!C!ED r>c$|j|Sr)rr#s r?rzNestedLinear.forwards!!!$$r>r\rs@r?rrsF%r>rc$eZdZfdZdZxZS) SkipModelct|tjdddj |_t j |_tt||_ y)NrFr)r) rrr_rrlinearr linear_skipr$rr)rJ double_nestrs r?rzSkipModel.__init__sP iiBU388: %<,,.!,"EFr>cl|j|}|j|}|j|}|Sr)rrrr#s r?rzSkipModel.forwards4 KKN   Q    q !r>r\rs@r?rrsG r>r)FT)FFr)r=r) contextlibrr3r&r|abcrrrcopyrenumrr functoolsr typingr r r rrrrrrunittestrr\torch.distributedrLrltorch.nnr_torch.nn.functionalrrtorch.distributed._composabler"torch.distributed._composable.fsdpr4torch.distributed._composable.fsdp._fsdp_param_grouprrtorch.distributed.device_meshrtorch.distributed.fsdprrr$torch.distributed.fsdp._common_utilsr"torch.distributed.fsdp._init_utilsr2torch.distributed.fsdp.fully_sharded_data_parallelrrr *torch.distributed.fsdp.sharded_grad_scalerr!torch.distributed.fsdp.wrapr"r#r$torch.distributed.tensorr%r&r'!torch.distributed.tensor.parallelr(r)r*r+r,r-torch.nn.parallel.distributedr.r`*torch.testing._internal.common_distributedr/r0r1r2$torch.testing._internal.common_utilsr3r4torch.utils._tritonr5r7rAr`rFrr}rrrrrrrrrrrr,r/r?rTrXr^r}rrrcontextmanagerrrrrrrrrrrrrtyperrrrr=r>r?rs #"    4:5O>R IRRFF FH P*44 BIIs 4 99$$B# 99##" " 2299 2t 23ryy33DBIID$D.Q"-Q"h[J-[J|$7D]J.]J@C mC L ? .,ryy,FJ*FJR/"))/@*r}}*Z'299'6 666 9H99 *X** .x.. .x.. :8:: >X>>0!00  0  0.+- TyyTIITc3h TB/&P#Pf 0(4.0f %299 %  r>