
    sg,                        d dl Z d dlZd dlZd dlZd dlZd dlZd dlmZ d dlm	Z
 d dlmZmZmZmZ  ej                   d      Z ej$                         Z ej(                  d      Zej-                  e       ej/                  e       ej1                  ej2                         de_         G d d	e      Zdd
ee   defdZdd
ee   defdZ dee   defdZ! G d d      Z"	 ddee"   dedefdZ#y)    N)Lock)default_timer)AnyListOptionalSequencestrobelight_function_profilerzB%(name)s, line %(lineno)d, %(asctime)s, %(levelname)s: %(message)sFc                       e Zd ZdZy)StrobelightCLIProfilerErrorzC
    Raised when an error happens during strobelight profiling
    N)__name__
__module____qualname____doc__     [/var/www/html/venv/lib/python3.12/site-packages/torch/_strobelight/cli_function_profiler.pyr   r      s    r   r   pidreturnc                 ~    d}| xs t        j                         } t        j                  |j                  |             S )zFReturns the link to the process's namespace, example: pid:[4026531836]z/proc/{}/ns/pid)osgetpidreadlinkformat)r   PID_NAMESPACE_PATHs     r   _pid_namespace_linkr   !   s3    *

C;;)00566r   c                     | xs t        j                         } t        |       }t        ||j	                  d      dz   d       S )z"Returns the process's namespace id[   )r   r   r   intfind)r   links     r   _pid_namespacer#   (   s;    

Cs#DtDIIcNQ&,--r   commandc                 $    dj                  |       S )N )join)r$   s    r   _command_to_stringr(   /   s    88Gr   c                       e Zd ZdZ e       Zddddddddddd	
d
ededede	dedede
ee	      de
ee	      dedefdZd"dZd#deddfdZd"dZd"dZdeddfdZdefdZdeded edefd!Zy)$StrobelightCLIFunctionProfilera  
    Note: this is a Meta only tool.

    StrobelightCLIFunctionProfiler can be used to profile a python function and
    generate a strobelight link with the results. It works on meta servers but
    does not requries an fbcode target.
    When stop_at_error is false(default), error during profiling does not prevent
    the work function from running.

    Check function_profiler_example.py for an example.
    FiX  g    cAzpytorch-strobelight-ondemand<   N   )
stop_at_errormax_profile_duration_secsample_eachrun_user_nametimeout_wait_for_running_sectimeout_wait_for_finished_secrecorded_env_variablessample_tagsstack_max_lenasync_stack_max_lenr-   r.   r/   r0   r1   r2   r3   r4   r5   r6   c       
             || _         || _        || _        || _        || _        || _        d | _        d | _        || _        y N)	r-   r.   r/   r0   r1   r2   current_run_idprofile_resultr4   )selfr-   r.   r/   r0   r1   r2   r3   r4   r5   r6   s              r   __init__z'StrobelightCLIFunctionProfiler.__init__C   sN     +(@%&*,H)-J* .237&r   r   c                    t        j                         }t        |      }ddddddddt        | j                         d	t        | j
                  d
z         d| d| g}| j                  r;|j                  d       |j                  dj                  | j                               t        j                  dt        |             t        j                  |d      }|j                  j                  d      }t        j                  d|       |j                   dk7  rt#        d|       t%        j&                  d|      x}r t        |j)                  d            | _        y t#        d|       )Nstrobeclientrunz
--profilerpyperfz--eventcyclesz--asyncz--sample-intervalz--duration-msi  z--pid:z--sample-tags,running command: %sTcapture_outpututf-8output:
{%s}r   z:failed to start strobelight profiling, error in run_async:zINFO Run Id: (-?\d+)r   z9failed to start strobelight profiling, unexpected result )r   r   r#   r    r/   r.   r4   appendr'   loggerdebugr(   
subprocessr?   stderrdecode
returncoder   researchgroupr9   )r;   	processId	namespacer$   resultoutputmatchs          r   
_run_asyncz)StrobelightCLIFunctionProfiler._run_async]   sW   IIK	"9-	4##$%4004789k9+&
  NN?+NN388D$4$456*,>w,GH=%%g._f-!-LVHU  II5v>>5>"%ekk!n"5D)GxP
 	
r   counterc                 D   |dkD  rt        d      ddd| j                   g}t        j                  dt	        |             t        j                  |d      }|j                  j                  d	      }t        j                  d
|       |j                  dk7  rt        d|       t        j                  d|      x}rU|j                  d      }|dk(  ry |dk(  r*t        j                  d       | j                  |dz          y t        d| d      t        d| d      )N   z*wait_for_running called more than 20 timesr>   getRunStatus--run-idrD   TrE   rG   rH   r   zAfailed to start strobelight profiling, error in wait_for_running:zProfile run status: (.*)r   RUNNING	PREPARING
   zunexpected z phaseunexpected output
: r&   )r   r9   rJ   rK   r(   rL   r?   rM   rN   rO   rP   rQ   rR   timesleep_wait_for_running)r;   rY   r$   rU   rV   rW   current_statuss          r   rd   z0StrobelightCLIFunctionProfiler._wait_for_running   s#   R<-<  ">:$BUBUAVX*,>w,GH=%%g._f-!-STZS[\  II8&AA5A"[[^N*;.

2&&w{31K?Ov2VWW),A&*KLLr   c                    dddt        | j                        g}t        j                  dt	        |             t        j                  |d      }|j                  j                  d      }t        j                  d|       |j                  d	k7  rt        d
|       t        j                  d|      x}r2|j                  d      }|j                  d      ry t        d| d      t        d| d      )Nr>   stopRunr]   rD   TrE   rG   rH   r   z<failed to stop strobelight profiling, return code is not 0 :INFO ::1:(.*)r   zSuccess!z*failed to stop strobelight profiling, got z resultra   r&   )strr9   rJ   rK   r(   rL   r?   rM   rN   rO   r   rP   rQ   rR   __contains__)r;   r$   rU   rV   rW   re   s         r   	_stop_runz(StrobelightCLIFunctionProfiler._stop_run   s    !9j#d>Q>Q:RS*,>w,GH=%%g._f-!-NvhW  IIov6656"[[^N**:61@@PPWX  *,A&*KLLr   c                    dddt        | j                        g}t        j                  dt	        |             t        j                  |d      }|j                  j                  d      }t        j                  d|       |j                  d	k7  rt        d
|       t        j                  d|      x}rg|j                  d      }|j                  d      r&t        j                   d       | j#                          y |j                  d      st        d|       g | _        t        j&                  d|      D ]2  }| xj$                  |d	   z  c_        t        j)                  |d	          4 y )Nr>   r\   r]   rD   TrE   rG   rH   r   z<failed to extract profiling results, return code is not 0 : rh   r   zProfile run status: PROCESSINGr`   z!Profile run finished with SUCCESSz9failed to extract profiling results, unexpected response zF(Total samples(.*)|GraphProfiler(.*)|Icicle view \(python stack\)(.*)))ri   r9   rJ   rK   r(   rL   r?   rM   rN   rO   r   rP   rQ   rR   rj   rb   rc   _get_resultsr:   findallinfo)r;   r$   rU   rV   rW   re   items          r   rm   z+StrobelightCLIFunctionProfiler._get_results   sK   !>:s4CVCV?WX*,>w,GH=%%g._f-!-NvhW  IIov6656"[[^N**+KL

2!!##001TU1OPVxX  !JJU
 	!D 47*KKQ 	!r   collect_resultsc                     	 | j                          t        j                  d       t        j                  d       |sy | j	                          y # t
        $ r!}t        j                  dd       Y d }~y d }~ww xY w)Nzstrobelight profiling stoppedzcollection stoppedzerror during stop_strobelightTexc_info)rk   rJ   ro   rK   rm   	Exceptionwarning)r;   rq   errors      r   _stop_strobelight_no_throwz9StrobelightCLIFunctionProfiler._stop_strobelight_no_throw   sd    	KNNKK78LL-." 	KNN:TNJJ	Ks   <A A 	A:A55A:c                 :   d}	 | j                          d}t        j                  d| j                         | j	                          t        j                  d       y# t
        $ r5}t        j                  dd       |r| j                  d       Y d }~yd }~ww xY w)NFTzstrobelight run id is: %szstrobelight profiling runningzerror during start_strobelight:rs   rq   )rX   rJ   ro   r9   rd   ru   rv   rx   )r;   strobelight_startedrw   s      r   _start_strobelightz1StrobelightCLIFunctionProfiler._start_strobelight   s    #	OO"&KK3T5H5HI""$KK78 	NN<tNL"///F		s   AA 	B%+BBwork_functionargskwargsc                 `   d | _         d | _        t        j                  j	                  d      x}r$|s4| j
                  rt        d      t        j                  d        ||i |S | j                         }|s]| j
                  r)t        j                  j                          t        d       ||i |}t        j                  j                          |S 	 t        j                  d       t               } ||i |}t               }||z
  }	t        j                  d|	       | j                  d       t        j                  j                          |S y # t        $ rN}
t        j                  dd	       | j                  d       t        j                  j                          |
d }
~
ww xY w)
NFzconcurrent runs not supportedz%failed to start strobelight profilingzcollection startedzwork function took %s secondsTrz   zwork function throw exceptionrs   )r9   r:   r*   _lockacquirer-   r   rJ   rv   r|   releaserK   timerro   rx   ru   )r;   r}   r~   r   lockedstartedrU   startend
total_timerw   s              r   profilez&StrobelightCLIFunctionProfiler.profile   s   ""399AA%HH6H%%56UVV>?$d5f55--/G%%288@@B5?  '77.44<<>12&77g 5[
;ZH///E.44<<>9 I:  >N///F.44<<>	s   A=E 	F-A	F((F-)r   N)r   )r   r   r   r   r   r   boolr    floatri   r   r   r<   rX   rd   rk   rm   rx   r|   r   r   r   r   r   r*   r*   3   s   
 FE
 $(/ ;,.-/6:+/ #&' ' #&	'
 ' ' '*' (+' !)c 3' d3i(' ' !'4'
RM MT M<M.!>KK 
K&D  %S % % % %r   r*   profilerr   c                 F      st        di | dt        dt        f fd}|S )Nr}   r   c                 l     t        j                         dt        dt        dt        f fd       }|S )Nr~   r   r   c                  0     j                   g| i |S r8   )r   )r~   r   r   r}   s     r   wrapper_functionz@strobelight.<locals>.strobelight_inner.<locals>.wrapper_function1  s     #8##MCDCFCCr   )	functoolswrapsr   )r}   r   r   s   ` r   strobelight_innerz&strobelight.<locals>.strobelight_inner0  s@    		'	DC 	D3 	D3 	D 
(	D  r   r   )r*   r   )r   r   r   s   `  r   strobelightr   *  s0     1;F;      r   r8   )$r   loggingr   rP   rL   rb   	threadingr   timeitr   r   typingr   r   r   r   	getLoggerrJ   StreamHandlerconsole_handler	Formatter	formattersetFormatter
addHandlersetLevelINFO	propagateru   r   r    ri   r   r#   r(   r*   r   r   r   r   <module>r      s     	 	    ) 0 0 
		:	;'''')GH	   Y '   / "   ) 7Xc] 7c 7. . . # o op :>56ILr   