sg?ddlZddlmZmZmZmZmZddlZddlm Z ddl m Z m Z gdZ ejeZ dZGddZGd d eZeej(d d ZdZGd dZGddZdZ ddeedfdeeeefdedeeedfdeeeefdeeeeeff dZdeefdZy)N)AnyDictListOptionalTuple map_aggregate) tree_flattentree_unflatten)TensorChunkSpecsplit_args_kwargs_into_chunks merge_chunksFceZdZdZdZy)_CustomReducera$ Custom reducer class that can be used to specify a custom operation that reduces losses of multiple microbatches into one value. Example: >>> # xdoctest: +SKIP >>> sum_reducer = _CustomReducer( >>> torch.tensor(0.0), >>> lambda a, b: a + b >>> ) c ||_||_yN) init_value reduce_fn)selfrrs Z/var/www/html/venv/lib/python3.12/site-packages/torch/distributed/pipelining/microbatch.py__init__z_CustomReducer.__init__(s$"N)__name__ __module__ __qualname____doc__rrrrrs  #rrc eZdZy) _LossReducerNrrrrrrrr-rrgc ||zSrr)abs rr%1s 1q5rcneZdZUdZdZeed<dZdZe de edffdZ e de e effd Zy ) r z2 Class used to specify chunking of inputs c||_yr split_dim)rr)s rrzTensorChunkSpec.__init__=s "rr)c||jjd|jjd|jdS)N.()) __class__rrr)rs r__repr__zTensorChunkSpec.__repr__Bs9~~(()4>>+B+B*C1T^^DTTU V rc"d|jdS)NzTensorChunkSpec(r-r(r/s r__str__zTensorChunkSpec.__str__Gs!$..!133r chunk_dims.c t|d}|S)a A helper for creating a tuple of `TensorChunkSpec` from a tuple of chunk dimensions (int's). Example: >>> # xdoctest: +SKIP >>> # There are three positional arguments to the model, and >>> # we are chunking them along dimension 0, 0 and 1, respectively >>> args_chunk_spec = TensorChunkSpec.from_tuple((0, 0, 1)) ct|Srr dims rr%z,TensorChunkSpec.from_tuple..Y ,rr)r3args_chunk_specs r from_tuplezTensorChunkSpec.from_tupleJs(  , rc t|d}|S)a\ A helper for creating a dictionary of `TensorChunkSpec` from a dictionary of chunk dimensions (int's). Example: >>> # xdoctest: +SKIP >>> # Chunk dimension 0 for the "id" argument, 1 for the "mask" argument >>> kwargs_chunk_spec = TensorChunkSpec.from_dict({"id": 0, "mask": 1}) ct|Srr6r7s rr%z+TensorChunkSpec.from_dict..kr9rr)r3kwargs_chunk_specs r from_dictzTensorChunkSpec.from_dict]s*  , ! rN)rrrrrint__annotations__r0r2 staticmethodrr;rstrr?rrrr r 8se#N 4#s(O$!cN!!rr c eZdZy) _ReplicateNr rrrrErEqr!rrEc i}g}|}d}t|t|k(s;Jdt|jdt|j|jD]G\}}t |\} } |j | ||} | Jt | \} } t| t| k7rt d|d| g}t| | D]\}}|tust|tjs|j |g|z?t|trqt|tjs J|d|j|j}||kr9|r"tj!d|d |d |d |}nt#d |d |d|dtj$|||j}t&rg}d}|D]}tj(|}||j|jz}t+dddg|j,z}t+||||j<|||<|j |||j|jz }|j |n|j |d}t/d||||<Jg}t1|D]O}i}|jD]'\}}g}|D]}|j |||||<)|j |Qg}|D]b} i}!t|t| k(sJt| j|D]\\}}}"t3||"|!|<|j |!d|S)aW Given a dictionary of args, and a dictionary of chunking specs, shard the args according to the chunking specs. Args: args_dict: Dictionary of args args_chunk_spec: Dictionary of chunking specs num_chunks: Number of chunks to shard the args into Returns: args_split: List of sharded args Tzargs_dict.keys() = z args_chunk_spec.keys() = NzArgument value z9 did not have the same number of values as as chunk spec z is not a tensorz%Tensor size on chunking dimension is z', downsizing the number of chunks from z to r+zArg z% on chunking dimension has a size of z$, smaller than the number of chunks z. PiPPy cannot reduce the number of chunks because other arguments have bigger chunk-dimension sizes. Please adjust your num_chunks setting.rFzUnrecognized chunk spec: )lenlistkeysitemsr append ValueErrorziprE isinstancetorchTensorr sizer)loggerwarning RuntimeError tensor_split_debug_mask_minibatches zeros_likeslicendim TypeErrorranger )# args_dictr: num_chunksargs_sharded_replicated arg_specsreal_num_chunks first_tensorarg_keyargflatspec chunk_specchunk_spec_flat_sharded_arg_flatvchunk_vv_split_dim_size chunk_tensorsexpanded_chunks split_dim_idx chunk_tensornew_val upper_idx slice_indices chunks_flat chunk_idx chunk_argskeyarg_single_chunkv_flat args_splitchunkper_chunk_argsarg_specs# r_shard_dict_of_argsr~us#(!I OL y>S n T).."2344NtTcThThTjOkNlmn ")I< !#& d$W- %%%)*5 t9O, ,!#'++5,8  dO48 GJAw*$Jq%,,,G ''o(=>G_5"!U\\2Jqc9I4JJ2#$66'*;*;#< #o5#CDTCUVDDN .'..w7%):):7;L;L)MM  N%++O<$++M:$ ";G9 EFFq8 Gt,<(SIreturnc  |i}|ttft|z}|#tj |tt}t tt |tt ||}t|}t |||}t||krz0split_args_kwargs_into_chunks..YsN! 1 Ns) r DEFAULT_CHUNK_DIMrGdictfromkeysr~ enumeraterTrKtupler[) rrrr:r>args_split_dictr` kwargs_splitrzrvs @rr r sYp~*+<=?#d)K  MM&/BS2TU) Yt_ Y '(O /*O&L  <?*l+- 4 ! ?+ ,    ?s<00 I?#$Bs<'8&9 ;  J%P %NuS_7MNNOP | ##rc|t|\}}n-t|d\}}ttgt|z}g}|D]I}t|\}}t|t|k7rt d|d||j |Kg} t |D]\} } t| trftt|D cgc] } || |  } } tr| dj}| ddD]}|j|k(rJtjtj|ddit| | j}g}d}t| t|k(sJt| |D]o\}}||j!| jz}t#dddg|j$z}t#|||| j<||}|j ||}qn| }| j tj&|| j ~t| t(rP| j*}tt|D]} | j-||| | }| j ||d| }tdt|D]} || | |k(rJ| j | t/| |Scc} w) z Given a list of chunks, merge them into a single value according to the chunk spec. Args: chunks: list of chunks chunk_spec: Chunking spec for the chunks Returns: value: Merged value NrzChunk z did not match chunk spec devicemeta)sectionsr8r7)r r rrGrLrKrrNr[rVshaperOrUemptyr)rMrQrXrYcatrrrr )rrfspec_flattened flatten_spec chunk0_flatchunks_flattenedr{chunk_flattenedrhargs_flattenedarg_idxrcrupartial_values overall_shapeval meta_chunks values_to_catchunk_start_idx partial_value meta_chunk chunk_end_idxrssliced reduced_valvalues rrr^sZ'3J'?$ %1$;! \)*;<=K@PP1)%0  3~#6 6veW,FzlST T0 1N!.10)  c? +"'s+;'v> 0  !# "#>*c+.>>>>14^[1Q4-M:$3joocmm6T$TM%*4t%<$= @R@R$RM38-3XM#--0*=9F!((0&3O4!/  ! !%))Ms}}"M N ^ ,..K"3'7#89  !mm!1)!rs;33 '<    8 $  ##$ > <5<<,.@A 5!5!r  ~J>B>B e$ S/e$ T#s(^ $e$ e$eOS$89: e$  S/%9 :; e$  4;T "# e$Pw8 Iw8r