sg~ NddlZddlZddlZddlZddlmZddlmZmZddl m Z m Z ddl m Z mZmZmZmZmZmZmZmZmZddlmZddlZddlmZmZddlmZdd lmZm Z dd l!m"Z"m#Z#dd l$m%Z%m&Z&dd l'm(Z(m)Z)erdd l*m+Z+e,ejZj]dddk(rdndZ/dZ0dgZ1Gdde2eZ3Gdde2eZ4Gdde3Z5Gdde4Z6GddZ7GddZ8de2d e,fd!Z9d"e,d#e,d$e,d ee:e,ffd%Z;d&eejxee2e,ffde2d dfd'Z=d&eejxee2e,ffde2d dfd(Z>d)ee4eeejxee2e,fffde2d dfd*Z?d)ee4eeejxee2e,fffde2d dfd+Z@Gd,d-eZAGd.de#ZBy)/N)deepcopy)autoEnum)partialwraps) AnyCallableDictListOptionalSetTupleType TYPE_CHECKINGUnion)Self)nnoptim) ModTracker)!register_optimizer_step_post_hook register_optimizer_step_pre_hook)is_traceable_wrapper_subclassTorchDispatchMode) tree_flatten tree_map_only)WeakIdKeyDictionaryweakref)RemovableHandlePYTORCH_NO_CUDA_MEMORY_CACHINGiTotal MemTrackerceZdZdZy)_RefTypeziBase Class for defining memory reference types, categorizing tensors based on their usage within a model.N__name__ __module__ __qualname____doc__W/var/www/html/venv/lib/python3.12/site-packages/torch/distributed/_tools/mem_tracker.pyr$r$2ssr+r$ceZdZdZy)_Statez;Base Class for defining module state to capture snapshots .Nr%r*r+r,r.r.6sEr+r.c,eZdZdZdZdZdZdZdZdZ dZ y ) _MemRefTypea An enum to define memory reference types, categorizing tensors based on their usage within a model. - PARAM: Tensors registered as nn.Parameter within modules. - BUFFER: Tensors registered as nn.Buffer within modules. - GRAD: Gradients associated with parameters. - ACT: Tensors produced during the forward pass and recomputation in activation checkpointing. - TMP: Temporary memory used during the backward pass, including gradients of activations. - OPT: Tensors holding optimizer states. - OTH: Tensors registered via `track_external` that do not fit the above categories. ParameterBufferGradient ActivationTempOptstateOtherN) r&r'r(r)PARAMBUFFERGRADACTTEMPOPTOTHr*r+r,r0r0:s,  E F D C D C Cr+r0c0eZdZdZdZdZdZdZdZdZ dZ d Z y ) _ModStatea An enum to define the state of a module. - PRE_FW: The module is about to run the forward pass. - POST_FW: The module has finished running the forward pass. - PEAK_FW: The module has reached the peak memory usage during the forward pass. - PRE_BW: The module is about to run the backward pass. - PRE_FW_AC: The module is about to run the forward pass with activation checkpointing. - POST_FW_AC: The module has finished running the forward pass with activation checkpointing. - POST_BW: The module has finished running the backward pass. - PEAK_BW: The module has reached the peak memory usage during the backward pass. z Pre-Forwardz Post-Forwardz Peak-Forwardz Pre-BackwardzPre-Forward-ACzPost-Forward-ACz Post-Backwardz Peak-BackwardN) r&r'r(r)PRE_FWPOST_FWPEAK_FWPRE_BW PRE_FW_AC POST_FW_ACPOST_BWPEAK_BWr*r+r,r@r@Ps1 FGG F I"JGGr+r@ceZdZdZdefdZy) _ModMemStatsa A class to store the memory statistics of a module. Args: mod_fqn (str): The fully qualified name of the module. Attributes: mod_fqn (str): The fully qualified name of the module. parameter_mem (int): The memory usage of the parameters of the module. buffer_mem (int): The memory usage of the buffers of the module. input_mem (int): The memory usage of the inputs to the module. output_mem (int): The memory usage of the outputs from the module. snapshots (Dict[_ModState, Dict[torch.device, Dict[str, int]]]): A dictionary of memory snapshots of the module at different states defined by ``_ModState``. Note: The memory snapshot is stored as a dictionary - Dict[torch.device, Dict[str, int]], where each key is a device, and each value is another dictionary with keys as memory reference types defined by `_MemRefType` and values as the memory consumed in bytes. mod_fqnc>||_||||i|_i|_yN)rK local_peak snapshots)selfrKs r,__init__z_ModMemStats.__init__|s#     35TVr+N)r&r'r(r)strrQr*r+r,rJrJhs&WWr+rJcPeZdZdZdededej deddf dZdefd Z d ejdefd Z e d ejdeejfd Ze dd ejdej dedeeeej*gefdeeej*ff dZy) _WeakRefInfozN Manages memory statistics and device attributes for tensor storages. size element_sizedevicereftypereturnNcf||_||_||_||_|j |_y)a Initializes the ``_WeakRefInfo`` object with tensor storage properties. Args: size (int): The number of elements in the tensor storage. element_size (int): The size of each element in the tensor storage. device (torch.device): The device on which the tensor is allocated. reftype (_RefType): The reference type of the tensor. N)rUrVrXrW_calculate_mem_consumed mem_consumed)rPrUrVrWrXs r,rQz_WeakRefInfo.__init__s2 (   88:r+c|j|jz}|jjdk(r#t j |t z t zS|S)z Calculates the memory consumed by the tensor storage, considering device-specific allocation rules. Returns: int: The memory consumed in bytes. cuda)rUrVrWtypemathceil_PYTORCH_MIN_ALLOCATE)rPmems r,r[z$_WeakRefInfo._calculate_mem_consumedsJii$+++ ;;  v %99c%::;>SS S r+stc|j|jk7r*|j|_|j|_|jS)a Updates and returns the memory consumed if the storage size has changed. Args: st (torch.UntypedStorage): The tensor storage to check for size updates. Returns: int: The updated memory consumed in bytes. )rUr[r\)rPrds r,update_mem_consumedz _WeakRefInfo.update_mem_consumeds@ 779 ! DI $ < < >D    r+tc |g}t}t|dkDr|j}t|r=|j \}}|j |Dcgc]}t ||c}nTt|ds)tjdt|tdn|j|jt|dkDr|Scc}w)z Recursively extracts untyped storages from a tensor or its subclasses. Args: t (torch.Tensor): The tensor to extract storages from. Returns: Set[torch.UntypedStorage]: A set of untyped storages. runtyped_storagezEExpected a tensor or a traceable wrapper-subclass of tensor, but got )category stacklevel)setlenpopr__tensor_flatten__extendgetattrhasattrwarningswarnr_ UserWarningaddri)rgunflattened_tensorsflattened_tensor_storagesobjattrs_attrs r,get_untyped_storagesz!_WeakRefInfo.get_untyped_storagess !c$'E!%&*%))+C,S1113q#**5+Q4GC,>+QRs$56MM_`deh`i_jk!,#$ .11#2E2E2GH%&*)(,RsCcallbackc||j|j||}tj||r t ||nd}||fS)a Creates a new ``_WeakRefInfo`` instance and a weak reference to a ``torch.UntypedStorage`` object, optionally attaching a callback to the weak reference. Args: st (torch.UntypedStorage): The storage object for which to create the weak reference info. device (torch.device): The device associated with the storage object. reftype (_RefType): The type of reference, used to categorize the storage. callback (Optional[Callable[[Self, weakref.ref]]]): A callback function that is called when the storage object is about to be finalized (garbage collected). The callback function should accept two arguments: the ``_WeakRefInfo`` instance and the weak reference to the storage. Returns: Tuple[Self, weakref.ref]: A tuple containing the newly created ``_WeakRefInfo`` instance and the weak reference to the storage object. The weak reference may have an attached callback if provided. N)rUrVrrefr)clsrdrWrXrwinfow_sts r, create_winfoz_WeakRefInfo.create_winfosF0BGGIr0&'B{{28wx7Nd{r+rM)r&r'r(r)inttorchrWr$rQr[UntypedStoragerf staticmethodTensorr r~ classmethodr r rrrrrrr*r+r,rTrTs ;;'*;49LL;KS; ;$   !e&:&: !s !) )U5I5I1J))8 BF      8T7;;$7$<=>  tW[[ ! r+rTunitsrYc ~ddddd}||vr||Std|ddj|j) Nr iii@)BKiBMiBGiBzUnsupported unit: z. Supported units are: z, ) ValueErrorjoinkeys)r unit_dicts r,_get_mem_divisorrsReEBI  '>tyyIY?Z>[ \  r+valuedivisor precisionc.|dk(r|St||z |S)Nr )round)rrrs r, _rounding_fnrsqL5GeEGOY&GGr+snapshotct|dk(r tdyt|jD]G\}}t |t ddkrtd|gfd|jDddiIy)NrNo memory tracked.rjzDevice: c 3PK|]\}}d|dt|ddyw) z: rj Nr).0kvrrs r, z"_print_snapshot.. s:AqQCr,q'156aw?s#&sep )rnprintritemsr _TOTAL_KEY)rrdevdev_snaprs ` @r,_print_snapshotrs 8} "#u%G!)  X ,gq 9Q >  se  $NN,     r+cX t|dk(r tdy ddlm}t  g}t t t|jj}dg|Dcgc]}|c}z}|jD]c\}} t| t ddkrt|g} | j fd| jD|j| et|||d y#t$r}td|d}~wwxYwcc}w) Nrrtabulate3Please install tabulate to use the tabulate option.Devicerjc3BK|]}t|ddyw)rjrNr)rrrrs r,rz*_print_snapshot_tabular..'s&Wl1gq12!E7;Wsrstheaderstablefmt)rnrr ImportErrorrlistnextitervaluesrrrrrRrqappend) rrrerr table_datakey_listkeyrrrrowrs ` @r,_print_snapshot_tabularrs 8} "#% u%GJDhoo/016689Hj9seH99G!) X ,gq 9Q > 3xj WX__EVWW#   (:w ?@  A  :sD 0 D' D$ DD$rOc|jD]B\}}t|t|D]#\}}td|dzdt||%Dty)Nz# r :)rr enumerater)rOrstate snapshot_listirs r,_print_state_snapshotsr,sf!* 1-} $]3 -KAx Bq1ugQ- He , --  Gr+c  ddlm}g}d}t|}|jD]\}}t |D]\} } |d| dz} | jD]s\} } t | t |ddkr| |k7r| ndt| d}| }| jD]\}}t ||dd |||<|j|ut||d d y#t$r}td|d}~wwxYw) Nrrrz # r rj)z State & Callrrrrr) rrrrrrrrRrr)rOrrrrlast_state_callrrrrr state_callrrrrrs r,_print_state_snapshots_tabularr7s?% JOu%G ) 1'}$]3 'KAx!7#a!eW-J!)!1 ' X 4gqAQF'1O&C !#h  #-$NN,KDAq%1!Wa%@$A5'"JC1#KK!!#& ' ''  (:v >?1  A  sC'' D0 C<<DcDeZdZeZeZeZeZy) _UpdateTypeN)r&r'r(rADDDELREFSIZEr*r+r,rrWs  &C &C &C 6Dr+rc eZdZdZd1dZ d2dededeedee ddf d Z d3d e jd e d e deefd Zdedej"ddfdZd1dZd1dZdeddfdZd e d e jddfdZ d4dedee j4eeefffdZ d5dej:de deeeffdZde defdZ!dej:de ddfdZ"dej:de de ddfdZ#dej:de ddfd Z$dej:de ddfd!Z%d e d"e&jNddfd#Z(d1d$Z)d1d%Z*d&e+ej:e&jNe jfddfd'Z, d6ded(ed)e ddfd*Z- d7d+ed(ed)e ddfd,Z.d1d-Z/d8fd. Z0de ddffd/ Z1d9d0Z2xZ3S):r"a A TorchDispatchMode to track, categorize and attribute the tensor memory created or accessed within its context. It categorizes the tracked tensors as parameters, buffers, activations, gradients, temporary memory and optimizer states as defined by ``_MemRefType`` within its context. It captures memory `snapshots` for the modules, called within its context, at various states defined by ``_ModState``. Attributes: memory_tracking: A weakref key dictionary to store the memory statistics of each module. Each key is a reference to a module, and each value is a ``_ModMemStats`` object that stores the memory statistics of the module. Note: The MemTracker should be used as a context manager. The modules, optimizers, and any other tensors created within the context of MemTracker will be tracked by default. Any tensors or stateful objects such as modules, optimizers etc. that need to be tracked but are created outside the MemTracker should be registered using the `track_external` method. The `track_external` method should be called before the MemTracker is used. Any tensors created outside the ``MemTracker`` and not supplied to the `track_external` method will not be tracked by the ``MemTracker``. Example usage: .. code-block:: python module = ... optimizer = ... inp = ... mem_tracker = MemTracker() mem_tracker.track_external(module, optimizer, inp) with mem_tracker as mt: loss = module(inp) print("After Forward:") mt.display_snapshot("current") loss.backward() optimizer.step() optimizer.zero_grad() mt.display_snapshot("peak") mt.display_modulewise_snapshots(depth = 3, units = "MiB") Known Limitations: - The ``MemTracker`` does not track memory for tensors that bypass the ``TorchDispatchMode`` ex. under ``no_dispatch``. - Resizing tensor storages directly by using non-Tensor methods other than using ``torch.Untyped_Storage.resize_`` is not tracked. File a Github issue if you have use-cases for this. - If the tensors are not traceable or wrappable subclasses of ``torch.Tensor``, then the tracker does not know how to track their storages. File a Github issue if you have use-cases for this. - During AC in the backward pass there might be misattribution between activation and temp memory, but the peak memory will be tracked accurately. This will be fixed in the next update by hooking intricately with ``torch.uitls.checkpoint``. rYNc2t|_i|_i|_i|_t|_d|_t|_t|_ t|_ d|_ d|_ d|_tj j"|_yNF)rmemory_tracking_curr_mem_snap _peak_mem_peak_mem_snap_param_to_grad_hook_handles_optimizer_hook_handles_WINFOr _mod_trackerr0 _ref_class_in_opt_in_ac_ac_modrrresize_ _orig_resizerPs r,rQzMemTracker.__init__s24BD24BD+>+@(  $*+ &L*5" ! .2 !0088r+u_typerold_mem_consumed old_reftypecd}|jj|jtj |j d}|jt d|tjk(r>||jxx|jz cc<|t xx|jz cc<n|tjk(r?||jxx|jzcc<|t xx|jzcc<d}n|tjk(r=|J||xx|jzcc<||jxx|jz cc<n_|tjk(r>|J|j|z }||jxx|z cc<|t xx|z cc<d}ntd||r<|j|jt dk(r|j|j=yyy)NFrTzInvalid update type: )r setdefaultrWdictfromkeysrrrrrXr\rrrr)rPrrrr maybe_zerorchanges r, _update_snapzMemTracker._update_snaps &&11 LL$--;  J* [__ $ U]] #u'9'9 9 # Z E$6$6 6 { & U]] #u'9'9 9 # Z E$6$6 6 J { &* ** [ !U%7%7 7 ! U]] #u'9'9 9 # {'' '#/ //''*::F U]] #v - # Z F * J4VH=> > ""5<<0<A'' 5B r+rgrXupdate_existingcFtj|}t}|D]}|jj |d\}}|L|j } | |k7r)||_|j tj|| |j|p|r tdtj||j||j\}} || f|j|<|jdkDr |j tj||j||S)NNN)rzNo existing winfo foundr)rTr~rmrgetrXrrrrwKeyErrorrrW_delete_callbackr\r) rPrgrXrstswinfosrdrr|rrs r,_update_and_maybe_create_winfosz*MemTracker._update_and_maybe_create_winfoss //2 "B{{r<8HE1 #mm ')$+EM%%koou+%V 5! 899+77'4+@+@ t$)$- B%%)%%koou= 5!1 "2 r+rcd|jdkDr!|jtj|yy)Nr)r\rrr)rPrrs r,rzMemTracker._delete_callbacks+    !   koou 5 "r+ctjdtjdtddffd }|tj_y)NrdrUrYc(j||jj|d\}}|^|j|jk7r@|j}|j |j tj||yyy)Nrr) rrrrUr\rfrrr)rdrUrr|rrPs r,rz)MemTracker._track_resize..resize_s   b$ '{{r<8HE1 UZZ2779%<#(#5#5 ))"-!!$$e>N"&= r+)rrrrrr)rPrs` r, _track_resizezMemTracker._track_resizesM t  ! ,, C D  " (/$r+cB|jtj_yrM)rrrrrs r,_restore_resizezMemTracker._restore_resizes'+'8'8$r+ peak_statec~|j}|jjD]}|j|jj vs&||j vs5|jD]`\}}|jj|d|tks-|t|j|<t||j |d|<b|jD]Z\}}|jj|d|tks-|t|j|<t||j|<\y)Nr)rrrrKrparentsrOrrNrrrrr)rPr curr_snap mod_statsrrs r,_update_peak_statszMemTracker._update_peak_statss) '' --446 I  D$5$5$=$==!4!44)2): X$//33C;hz>RR8@8LI005GO (HI// ;B?D '__. >MC~~!!#q)HZ,@@&.z&:s#+3H+=##C( >r+ctj|}|D]}|jj|d\}}|^|j|j k7r?|j }|j ||jtj||ytj||j||j\}}||f|j|<|j dkDs|jtj|y)Nrrr)rTr~rrrUr\rfrrrrrWrr) rPrXrgrrdrr|rrs r,_trackzMemTracker._track)s//2 >B{{r<8HE1 ::*','9'9$--b1%%#((%BR&*77'4+@+@ t$)$- B%%)%%koou=# >r+r_c|dk(rt|jS|dk(rt|jStd|)aE Capture a snapshot of the memory usage breakdown per device, based on the specified type. Args: type (str): The type of snapshot to capture. Can be "current" for the current memory usage or "peak" for the peak memory usage. Defaults to "current". Returns: Dict[torch.device, Dict[str, int]]: A dictionary where each key is a torch.device, and each value is another dictionary. This inner dictionary has keys representing memory reference types as defined in ``_MemRefType`` and values representing the amount of memory consumed in bytes. Raises: ValueError: If an invalid type is specified. currentpeakz Invalid type )rrrr)rPr_s r,get_tracker_snapshotzMemTracker.get_tracker_snapshotAsG" 9 D//0 0 V^D//0 0}TF34 4r+moduleinstall_grad_hooksc dtjddffd d}|jD]}j|tj }|t d|Dz }|j*j|jtjjj|d|s|j }|j fd}||fj|<d}|jD]7} j| tj}|t d|Dz }9||fS)NgradrYcFj|tjyrM)rr0r:)rrPs r, _grad_hookz?MemTracker._track_module_params_and_buffers.._grad_hookas  0 0   r+rc34K|]}|jywrMr\rrs r,rz>MemTracker._track_module_params_and_buffers..msGu 2 2Gc(|jSrM)r)prs r,z=MemTracker._track_module_params_and_buffers..ysz!&&1r+c34K|]}|jywrMrrs r,rz>MemTracker._track_module_params_and_buffers..s H!3!3 Hr)rr parametersrr0r8sumrr:rr register_hook"register_post_accumulate_grad_hookbuffersr9) rPrr param_memoryparamrgrad_hook_handlepost_acc_grad_hook_handle buffer_memorybufferrs ` @r, _track_module_params_and_buffersz+MemTracker._track_module_params_and_buffersYsP U\\ d   &&( E99!!F CGGG GLzz%44JJ$$ 0044UDAI&#(#6#6z#B ,1,T,T2-)%-;007' . nn& IF99""F S H HH HM  I m,,r+argscvddtjddffd }ttj||S)NrrgrYctj|}|D]3}jj|d\}}|%|jz 5y)Nr)rTr~rrr\)rgrrdrr|input_or_output_memoryrPs r,add_inps_or_outsz=MemTracker._track_inputs_or_outputs..add_inps_or_outssS33A6C A;;??2|<q$*e.@.@@* Ar+)rrr)rPr+r/r.s` @r,_track_inputs_or_outputsz#MemTracker._track_inputs_or_outputss<!" A  A A ell$4d;%%r+inputscl|jj|}|J||jvrgt|}|j |d\}}|j |}||_||_||_||j|<tj}n|jjrM|j|}tj}|jtj||_ d|_nwt#|jj$|hz } t'| dk(rd| vr t)d|j|}tj}|j |}||_|j+} |tjk(ri| j-D cic]\} } | | t.c} } |_|j2j5tj6gj9| |j2j5|gj9t;| ycc} } w)NTrr GlobalzMemTracker does not support memory tracking for multiple iterative calls. Either use ``reset_mod_stats`` to clear module memory stats for the previous iteration or file a github issue if you need this feature.)r get_known_fqnrrJr*r0 parameter_mem buffer_mem input_memr@rAis_bwrErrrrrmrrnNotImplementedErrorrrrrNrOrrCrr) rPrr1mod_namer  param_memr7r8rr mem_snapshotrrs r, _pre_fw_hookzMemTracker._pre_fw_hooks$$226:### -- -$X.I$($I$I4%J% !Iz55f=I&/I ##-I "+I +4D  ($$E    $ $,,V4I''E||#&{{62 " $++334zAG7|q X%8)H ,,V4I$$E55f=I"+I 002 I$$ $?K?Q?Q?S$.;c8Xj))$I     * *9+<+ MMAa P ((0 002 ;G;M;M;O *7#xC*% %  &&y'8'8"=DD\R&&y'7'7<CC \ "  s C&c|tjddy|j|}|jj t j gj|jy)Nz&Module is None. Skipping POST_BW hook.rjrD) rtrurrOrr@rGrr)rPrr+r s r, _post_bw_hookzMemTracker._post_bw_hooks^ > MMBq Q ((0 &&y'8'8"=DD  % % ' r+ optimizerc|jjD]D}|jD]/}t|tjs|j ||1FyrM)rr isinstancerrr)rPrXrHstatesvals r,_track_optimizer_statesz"MemTracker._track_optimizer_statessW oo,,. F}} c5<<088  r+cdtjdtdtddffd }dtjdtdtddffd }t|t |f_y)NrHr+kwargsrYcd_y)NT)rrHr+rOrPs r,_opt_step_pre_hookzFMemTracker._register_global_optimizer_hook.._opt_step_pre_hooks  DLr+cTjtj|d_yr)rMr0r=rrQs r,_opt_step_post_hookzGMemTracker._register_global_optimizer_hook.._opt_step_post_hooks   ( () D DLr+)r Optimizerrrrr)rPrRrTs` r,_register_global_optimizer_hookz*MemTracker._register_global_optimizer_hook ss  .1 ;>   ! !.1 !;> !  ! --? @ -.A B( $r+c&|jjD]%\}}|j|j'|jj|j)|jD]}|jd|_yyrM)rrremoveclearr)rPr&r'handles r,%_deregister_param_and_optimizer_hooksz0MemTracker._deregister_param_and_optimizer_hooks s - - 4 4 6 /  %  # # % % , , .  / ((..0  ' ' 366   +/D ( 4r+externalct|\}}|D]}t|tjr!|j |t j >t|tjjr|j|dvt|tjr!|jt j|tdt|dy)a Track tensors and stateful objects like modules, optimizers etc. that are created outside the MemTracker. This method should be called before the ``MemTracker`` is used. Any tensors that are not module parameters, buffers, gradients activations, or optimizer states will be categorized as ``Other``. If you want them categorized with a custom name, please file a GitHub issue. Any tensors created outside the MemTracker and not supplied to this method will not be be tracked by ``MemTracker``. Args: *external (Union[nn.Module, optim.Optimizer, torch.Tensor]): The external modules, optimizers, and tensors to be tracked. Fr3zObject of type zj is not supported for tracking. Only stateful objects like modules, optimizers, and tensors are supported.N)rrJrrrr0r>rModuler*rrUrMr= TypeErrorr_)rPr\ flat_externalr|rzs r,track_externalzMemTracker.track_external.s(1 q  C#u||,44OOC155ce5TC1,,[__cB%d3i[1ab r+rrc\|j|}|r t||yt||y)a' Display the memory usage breakdown snapshot of the tracker based on the specified type and units. Keyword args: type (str): The type of snapshot to display. Can be "current" for the current memory usage or "peak" for the peak memory usage. Defaults to "current". units (str): The units to use for displaying memory usage. Defaults to "B". Supports ["B", "KiB", "MiB", "GiB"]. tabulate (bool): Whether to display the snapshot in a tabular format. Defaults to False. N)rrr)rPr_rrrs r,display_snapshotzMemTracker.display_snapshotNs+,,T2  #He 4 He ,r+depthcndtdttttffdt |j j fdD]e}|j}|jddz}||kDr)td||rt|j|Pt|j|gy ) aZ Print per device memory breakdown snapshot for each module called within MemTracker. Snapshots are displayed for the states defined by ``_ModState``. The module hierarchy is displayed up to the specified depth. Keyword Args: depth (int, optional): The depth of the module hierarchy to display. Defaults to 2. units (str, optional): The units to use for memory tracking. Defaults to "B". Supports ["B", "KiB", "MiB", "GiB"]. tabulate (bool, optional): Whether to display the snapshot in a tabular format. Defaults to False. srYctjd|Dcgc]-}|jr t|n|j /c}Scc}w)Nz([0-9]+))resplitisdigitrlower)rftexts r,natural_sort_keyzAMemTracker.display_modulewise_snapshots..natural_sort_keyosFHHZ3"\\^D = s2Ac(|jSrM)rK)m_statsrms r,rz9MemTracker.display_modulewise_snapshots..ws 0 Ar+)r.r z Module: N) rRr rrsortedrrrKcountrrrOr)rPrdrrr rK mod_depthrms @r,display_modulewise_snapshotsz'MemTracker.display_modulewise_snapshots`s  U38_(=    ' ' )A CI ''G c*Q.I5  IgY' (.y/B/BEJ&y':':EB Cr+c8|jjy)z[ Reset all the module memory stats. Clears ``memory_tracking`` dictionary. N)rrYrs r,reset_mod_statszMemTracker.reset_mod_statss ""$r+c|j|jj|j|j|j |j |j|j|_ |jjDcic]\}}||tc}}|_ |jjt|5|Scc}}wrM)rVrregister_user_hooksr>rBrErGrrrrrr __enter__super)rPrr __class__s r,ryzMemTracker.__enter__s ,,. --              "779;?;N;N;T;T;V *7#xC*% %  ##%   sC!c|j|jj|jt |||jj |yrM)r[rclear_user_hooksrrz__exit__)rPr+r{s r,r~zMemTracker.__exit__sP 224 **,  $"""D)r+c||i|xsi}|jrtj}nC|jjr|j stj }ntj}ttjt|j|||jjrtjntj}|j!||SrM)rr0r=rr9rr<r;rrrrr r@rHrCr )rPfunctypesr+rOresrXrs r,__torch_dispatch__zMemTracker.__torch_dispatch__sD)FLb) <<!ooG    $ $T[[!&&G!ooGellGDKK$A3G*.*;*;*A*AY&&yGXGX   + r+)rYNr)F)r)T)rrF)rjrF)rYr")r*N)4r&r'r(r)rQrrTr rr$rrrboolr rrrrrrr.r r rRr rWrrr^rr*rr0r>rBrErGrrUrMrVr[rrarcrtrvryr~r __classcell__)r{s@r,r"r"ds.`92+/*. *6*6*6#3- *6 h' *6  *6`!& ! <<!! ! \  !F6l6'++6$6 / 9>V>>.>h>5<<>D>2$55 ellDcN* +52=A--ii--59-- sCx--^ &S &S &5Q2995Qc5Qd5QnVBIIVsVSVTV& 299 C D &  BII  S  T    ,1OO    ( 0ryy%//5<<GH BIN--,/-AE- -&BG!C!C%(!C:>!C !CF% "*c*d*r+)Cr`osrhrtcopyrenumrr functoolsrrtypingrr r r r r rrrrtyping_extensionsrrrr$torch.distributed._tools.mod_trackerrtorch.optim.optimizerrrtorch.utils._python_dispatchrrtorch.utils._pytreerrtorch.utils.weakrrtorch.utils.hooksrrenvironrrbr__all__rRr$r.r0r@rJrTrfloatrrWrrrrrr"r*r+r,rs& $   # ;<91  ?C D IDq .tsDtFS$F(,0WWOH d5<<c3h#?@  QU $A5<<c3h/0A9<A A4 FDellDcN&B!CDDE NQ   @FDellDcN&B!CDDE@NQ@ @@ $ K "K r+