
    sg9                     @   d dl Z d dlZd dlmZmZmZmZmZmZ d dl	Z	d dl
Z
d dlZd dlmZ d dlZd dlmZmZ d dlZd dlZ ej(                  e      Zd Zd Z e       Zd Zeee ed      ee fZ!dZ"d	 Z# G d
 de      Z$ddddZ%d Z&d Z'd Z(d Z)dZ*dZ+d Z,d Z-d Z.y)    N)AnyDictList
NamedTupleOptionalTuple)NamedTemporaryFile)_frames_fmt_block_extrac                      dfd}t        j                  |        fdt        j                  j	                         fd}|S )NTc                  
    d y )NF )enableds   J/var/www/html/venv/lib/python3.12/site-packages/torch/utils/viz/_cycles.pydisablez observe_garbage.<locals>.disable   s
         c                     sy | dk(  r$t        j                  t         j                         y | dk(  r6t        j                         dgfd}t        j
                  |       y y )NstartstopFc                     d   sdd<   nt        j                         d	 d   dk7  rt        j                           t        j                         t        j                  j                          t        j                  d       t        j                  j                         }t        j                          t        j                  j                         }||k7  rt        j                  d||z
         d | i |S y # dw xY w)Nr   TF
generation   z.CUDA Memory changed during GC, %d bytes freed.)sys
setprofilegccollectgarbageclear	set_debugtorchcudamemory_allocatedloggerwarning)	argskwargsbeforeafterr   infoobserver
orig_traceself_returns	       r   
do_collectz8observe_garbage.<locals>.gc_callback.<locals>.do_collect#   s    "1~%)KNNN:.#G'  -2JJL ,

((* Q!&!<!<!>

 %

 ; ; =!U?"NN+[]cfk]kl"&)%t6v66 * #'s   CD D	)r   r   DEBUG_SAVEALLr   
getprofiler   )phaser)   r-   r+   r,   r   r*   s    ` @@r   gc_callbackz$observe_garbage.<locals>.gc_callback   s[    GLL))*f_)J 'K7 7: NN:&C r   c                  D    t         j                  j                          y N)r   	callbacksremove)r1   s   r   r5   zobserve_garbage.<locals>.removeE   s    
K(r   )atexitregisterr   r4   append)r*   r   r5   r   r1   s   `  @@r   observe_garbager9      s?    G
 OOG''R LL$)Mr   c                  D    dd} t         |        j                  d         S )Nc                       fdS )Nc                       S r3   r   xs   r   <lambda>z+_get_cell_type.<locals>.f.<locals>.<lambda>Z   s    q r   r   r=   s   `r   fz_get_cell_type.<locals>.fY   s	    r   r   r3   )type__closure__)r@   s    r   _get_cell_typerC   X   s    "##r   c                     i fd fdfd}fd} fd} fd} fd}fd} fd	} fd
}fd}	t         |t        |t        |t        |t        |t
        j                  |t
        j                  |t        |t
        j                  |t        j                  |t
        j                  |	i}
t               j                  D ]  }||
v s |
|             dd       t         t              r d       S )a%  
    Return known information about references held by the given object.

    Returns a mapping from referents to lists of descriptions.  Note that there
    may be more than one edge leading to any particular referent; hence the
    need for a list.  Descriptions are currently strings.

    c                 Z    j                  t        |      g       j                  |        y r3   )
setdefaultidr8   )nameobj
referencess     r   add_referencez+annotated_references.<locals>.add_referencej   s"    bgr*11$7r   c                  T    | D ]"  }t        |      s |t        |             $ y r3   )hasattrgetattr)attrsattrrK   rI   s     r   	add_attrsz'annotated_references.<locals>.add_attrsm   s-     	8DsD!dGC$67	8r   c                  6    	   d       y # t         $ r Y y w xY w)Ncell_contents)
ValueErrorrQ   s   r   add_cell_referencesz1annotated_references.<locals>.add_cell_referencesr   s$    	o& 	
 	s    	c                  &      ddddddddd		       y )
N__defaults__rB   __globals____code____name__
__module____doc____qualname____annotations____kwdefaults__r   rU   s   r   add_function_referencesz5annotated_references.<locals>.add_function_references|   s'    .!#"		$r   c                  F    t              D ]  \  } } d|  d|        y )N[])	enumerate)positionitemrK   rI   s     r   add_sequence_referencesz5annotated_references.<locals>.add_sequence_references   s,    'n 	1NHdAhZq/40	1r   c                  t    j                         D ]$  \  } } d|         dt        |        d|       & y )Nkeyrb   rc   )itemsrepr)ri   valuerK   rI   s     r   add_dict_referencesz1annotated_references.<locals>.add_dict_references   s=    ))+ 	3JC%%Ad3i[*E2	3r   c                  &    D ]  }  d|         y )Nelementr   )eltrK   rI   s    r   add_set_referencesz0annotated_references.<locals>.add_set_references   s     	*C)S)	*r   c                        ddd       y )N__self____func__im_classr   rU   s   r   add_bound_method_referencesz9annotated_references.<locals>.add_bound_method_references   s    *j*5r   c                      t              t        j                  u r3t        j                        } t        |       dk(  r| d   } d|       y y y )N   r   __callback__)rA   weakrefrefr   get_referentslen)	referentstargetrK   rI   s     r   add_weakref_referencesz4annotated_references.<locals>.add_weakref_references   sN     9#((-I9~""1nf5 # $r   c                      j                   }  dddddd       t        |       t        u r/j                   j                         D ]  \  }} d| |        y y )Nf_backf_code
f_builtins	f_globalsf_tracef_localszlocal )r   rA   dictrj   )r   rH   localrQ   rK   rI   s      r   add_frame_referencesz2annotated_references.<locals>.add_frame_references   sa    <<(HlKJW >T!"||113 6etfou56 "r   c                        ddd       y )N__objclass__r[   __doc__r   rU   s   r    add_getset_descriptor_referencesz>annotated_references.<locals>.add_getset_descriptor_references   s    .*i8r   __dict__	__class____mro__)tuplelistr   set	frozensettypesFunctionType	FrameTypeCellType
MethodTyperz   r{   GetSetDescriptorTyperA   r   
isinstance)rI   rV   r`   rg   rm   rq   rv   r   r   r   type_based_referencestype_rQ   rK   rJ   s   `           @@@r   annotated_referencesr   _   s     (*J88

$13
*6669 	&%!%3-%5+""$D c"" +))(!%(*+ j+&#t)r       c                    d }t        | t              rt        |       S t        |       j                  dk(  rd| j                   S t        | t
        j                        r	 | j                  j                  }d| S t        | t              rd ||        dS t        | t              rd ||        d	S t        | t              rd
t        |        dS t        | t
        j                        rd| j                   S t        | t              rd| j                   S t        | t        j                         r |        }|ydt#        |      dS t        | t
        j$                        rJ| j&                  j(                  }t        |      t*        kD  rd|t*        dz
   d z   }d| d| j,                   S dt        |       j.                   dt        |       j                   S # t        $ r d}Y ww xY w)zx
    Return a string to be used for Graphviz nodes.

    The string should be short but as informative as possible.
    c                     dj                  d t        t        d      |       D              }t        |       dkD  r| dt        |       dz
   }|S )N,c              3      K   | ]7  \  }}t        |t              rt        |      nt        |      j                   9 y wr3   )r   
BASE_TYPESrk   rA   r[   ).0ir>   s      r   	<genexpr>z=object_annotation.<locals>.format_sequence.<locals>.<genexpr>   s3     rW[WXZ[:a#<Q$q'BRBRRrs   =?   z, ...)joinzipranger}   )rI   bodys     r   format_sequencez*object_annotation.<locals>.format_sequence   sN    xxr_bchijckmp_qrrs8a<V5SA/Dr   functionz	function
z<anonymous>zinstancemethod
rb   rc   ()zdict[zmodule
ztype
Nzweakref (dead referent)zweakref to id 0xr>   z...   zframe
:zobject
.)r   r   rk   rA   r[   r   r   rt   __qualname__AttributeErrorr   r   r   r}   
ModuleTyperz   r{   rG   r   r   co_filenameFRAME_FILENAME_LIMITf_linenor\   )rI   r   	func_namereferentfilenames        r   object_annotationr      s    #z"CyCyZ'CLL>**	C))	*	&11I ")--	C	?3'(**	C	?3'(**	C	s3xj""	C))	*#,,((	C	~&&	C	%5,%bl1%566	C	)::))x=//x*>*B(C(DEEH
!CLL>22$s)../qc1C1C0DEE3  	&%I	&s   !G   G/.G/c                   L    e Zd ZU eed<   ee   ed<   eed<   eeee	f      ed<   y)Nodelabelcontextroot
referrentsN)
r[   r\   r   strr^   r   boolr   r   intr   r   r   r   r     s)    Jc]
JU38_%%r   r   )r   filterc          
      h   |
t               }|t        }| D cg c]%  }t        t        |       ||       ||      g       ' }}| D cg c]  }g  }}t	        |       D ci c]  \  }}t        |      | }}}| D ]  }|t        |         }||   }	t        |      }
t        j                  |      D ]r  }t        |      }|j                  |d       }|#||   }|
j                  |dg      }||   j                  |       |D ]  }|	j                  j                  ||f       ! t  t	        |      D cg c]  \  }}|j                  s| }}}t               }|r?|j                         }||v r|j                  |       ||   }|j!                  |       |r?i }g }t	        |      D ])  \  }}||v st#        |      ||<   |j                  |       + |D ]7  }|j                  D cg c]  \  }}||v r|||   f c}}|j                  d d  9 |S c c}w c c}w c c}}w c c}}w c c}}w )N?)cuda_allocation_contextis_cuda_tensorr   r   rd   rG   r   r   r|   getr8   r   r   r   popaddextendr}   )objectsr   r   rI   nodesnode_referrersr   
id_to_nodefidxr@   rJ   	referrentridtidxtlabelsr   n	to_searchto_keepidx	referrersid_to_filtered_idfiltereds                           r   create_graphr     sZ   )+~T[\ST#C('#,sRH\E\5<&=cr&=N&=+4W+=>C"S'1*>J> 3"S'"$K)#.
))#. 		3IY-C>>#t,D|dA^^C#/F4 ''- 3##UDM23		3	3  )/:tq!166:I:eG
mmo'>C"3'	#  )+H%  1<#&'8#9a OOA  8/0||8+s!%66 "#4S#9: 8Q8 OM ]&=>  ; 8s#   *H		H"H"8H(H(.H.c                 ,    t        j                  |       S r3   )jsondumps)r   s    r   escaper   5  s    ::a=r   c                     t        | t        j                        xr3 | j                  xr% t        | t        j                  j
                         S r3   )r   r    Tensoris_cuda_subclasses
FakeTensor)rI   s    r   r   r   9  s8    c5<<(nS[[nCQVQbQbQmQmAn=nnr   c                      t         j                  j                  j                         } i | d   D ]4  }|d   }|d   D ]%  }|d   dk(  rt	        |      \  }}||<   ||d   z  }' 6 fd}|S )Nsegmentsaddressblocksstateactive_allocatedsizec                     t        |       rM| j                         j                         }j                  |      }|dj	                  t        |d            S y )N
T)full_filename)r   untyped_storagedata_ptrr   r   r
   )rI   addrframesaddr_to_frames      r   object_contextz/cuda_allocation_context.<locals>.object_contextG  sQ    #&&(113D"&&t,F!yyV4!HIIr   )r    r!   memory	_snapshotr   )snapshotsegr   blkr   	real_sizer   r   s          @r   r   r   <  s    zz  **,HM
#  9~x= 	 C7|11$0$5!	&,d#CKD		   r   c                 ~   g d}t        |       D ]@  \  }}|j                  | dt        |j                         d|j                  rdnd d       B t        |       D ]<  \  }}|j
                  D ](  \  }}|j                  | d| dt        |       d	       * > |j                  d
       dj                  |      S )N)zdigraph GraphName {znode [shape=rect];zrankdir=LR;z [label=z, color=redblackz];z -> z
 [label = rc   z}
r   )rd   r8   r   r   r   r   r   )r   linesr   r   r@   r   js          r   to_dotr  P  s    HE%  _1s(6!''?"38affURY;ZZ\]^_ %  B1 	BHE1LLA3d1#Zua@A	BB 
LL99Ur   az  
<!DOCTYPE html>
<html>
<head>
  <style>
    body {
      margin: 0;
      padding: 0;
      overflow: hidden;
    }

    #container {
      display: flex;
      flex-direction: column;
      height: 100vh;
    }

    #main {
      flex: 2;
      overflow: auto;
    }

    #preContainer {
      flex: 1;
      overflow: auto;
    }

    svg {
        overflow: scroll;
    }

    pre {
      margin: 0;
      padding: 10px;
    }
  </style>
</head>
<body>
  <div id="container">
    <div id="main">
    </div>
    <div id="preContainer">
      <pre id="stacktrace">Mouse over tensor objects to see where they were allocated.</pre>
    </div>
  </div>
<script src='https://cdnjs.cloudflare.com/ajax/libs/viz.js/1.8.0/viz-lite.js'></script>
<script>
let dot = $DOT
let image = Viz(dot, {format: 'svg'});
document.getElementById('main').innerHTML = image
$LISTENERS
</script>
</body>
</html>
z
document.getElementById('node{id}').addEventListener('mouseover', function(event) {{
  document.getElementById("stacktrace").textContent = {stack}
}})
c           	         g }t        |       D ]h  \  }}|j                  t        j                  t	        |dz         t        |j                   d|j                               }|j                  |       j t        |       }t        j                  dt        |            j                  ddj                  |            S )Nrx   z:
)rG   stackz$DOTz
$LISTENERSr   )rd   r   _listener_templateformatr   r   r   r8   r  	_templatereplacerk   r   )r   	listenersr   r   sdots         r   to_htmlr    s    I%  199%%QU6QWWISQRQZQZP[B\;]%^	
 -CVT#Y/77diiPYFZ[[r   c                 x     t         j                  j                  j                  d        fd}t	        |      S )Ni )max_entriesc                     | rCt        d | D              st        j                  d       y  t        t	        |                    y y )Nc              3   2   K   | ]  }t        |        y wr3   )r   )r   rI   s     r   r   z:observe_tensor_cycles.<locals>.observer.<locals>.<genexpr>  s     >s~c*>s   z No CUDA Tensors found in garbage)anyr#   r)   r  r   )r   callbacks    r   r*   z'observe_tensor_cycles.<locals>.observer  s;    >g>>>?W\'234	 r   )r    r!   r   _record_memory_historyr9   )r  r*   s   ` r   observe_tensor_cyclesr    s0    	JJ,,,@5 8$$r   c                  H    t         j                  d       d } t        |       S )a  
    Install a warning that reports whenever a cycle that is holding CUDA memory is observed.

    The warning produces an .html file that visualizes the cycle,
    and links it to the stack frame that allocted the CUDA tensor.

    Reference cycles are freed by the cycle collector rather than being cleaned up
    when the objects in the cycle first become unreachable. If a cycle points to a tensor,
    the CUDA memory for that tensor will not be freed until garbage collection runs.
    Accumulation of CUDA allocations can lead to out of memory errors (OOMs), as well as
    non-deterministic allocation behavior which is harder to debug.
    z2Watching Python reference cycles for CUDA Tensors.c                     t        ddd      5 }|j                  |        t        j                  d|j                         d d d        y # 1 sw Y   y xY w)Nwz.htmlF)suffixdeletezDReference cycle includes a CUDA Tensor see visualization of cycle %s)r	   writer#   r$   rH   )htmlr@   s     r   write_and_logz)warn_tensor_cycles.<locals>.write_and_log  sJ    GEB 	kaGGDMNNacdcicij	k 	k 	ks   2A

A)r#   r)   r  )r!  s    r   warn_tensor_cyclesr"    s$     KKDEk !//r   )/r   r   typingr   r   r   r   r   r   r   rz   r   tempfiler	   r    torch.cuda._memory_vizr
   r   r6   logging	getLoggerr[   r#   r9   rC   r   r   r   floatcomplexrA   r   bytesr   r   r   r   r   r   r   r   r  r  r
  r  r  r"  r   r   r   <module>r+     s    	 
 ? ?    '  <  			8	$8R$
 gZ 5'4:sE:
 .Fd&: & &*$ ,\o(	6	n 
\	%0r   