
    sg                     B   U d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	Z	d dl
Z
d dlZd dlZd dlZd dlZd dlmZ d dlmZ d dlmZ d dlmZ d dlmZmZmZ d dlmZ d dlmZmZm Z m!Z!m"Z"m#Z#m$Z$m%Z% d d	l&m'Z' d dl(Z(d dl)Z(d dl*Z(d dl+m,Z- d dl.m/Z/ d d
l0m1Z1m2Z2m3Z3m4Z4m5Z5m6Z6m7Z7m8Z8m9Z9m:Z: d dl;m<Z<m=Z=m>Z> d dl?Z? ej                  ej                          ej                  eC      ZD G d de      ZEi d eEdd      d eEdd      d eEdd      d eEdd      d eEdd      d eEdd       d! eEd"d#      d$ eEd%d&      d' eEd(d)      d* eEd+d,      d- eEd.d/      d0 eEd1d2      d3 eEd4d5      d6 eEd7d8      d9 eEd:d;      d< eEd=d>      d? eEd@dA      ZFe G dB dC             ZGdD ZHdE ZIdF ZJdG ZKdH ZLdI ZMdJ ZNdK ZOdL ZPdM ZQdN ZRdO ZSdP ZTdQ ZUdR ZVdS ZWdT ZXdU ZYe4dVdWdX edYZ      dXd[dXfd\       ZZe8rd]Z[n e\ ej                  d^d_            Z[d`daiZ^e7rdbe^dc<   dddZ_dee\fdfZ`edg        Zaddhe\die\dje\fdkZbdie\dlecfdmZddaee e	j                     egdn<   ddoe ec   dedfdpZhddqZidrZj G ds dte9      Zkdueece"e#   f   dve$dwe#fdxZldamdeenfdyZodz Zpde[ejfd{Zq G d| d}e9      Zr G d~ de/j                        Zt G d de/j                        Zuedd       Zv G d de(j                  j                  jr                        Zy G d dek      Zz G d de9      Z{y)    N)contextmanager)	dataclass)	timedelta)Enum)partialreducewraps)StringIO)Dict
NamedTupleOptionalUnionListAnyCallableTuple)patch)
FILE_SCHEMAfind_free_portIS_SANDCASTLEretry_on_connect_failuresskip_but_pass_in_sandcastleskip_but_pass_in_sandcastle_ifTEST_WITH_ROCMTEST_WITH_TSANTestCase	run_tests)_install_threaded_pg_uninstall_threaded_pgProcessLocalGroup)levelc                   "    e Zd ZU eed<   eed<   y)TestSkip	exit_codemessageN)__name__
__module____qualname__int__annotations__str     ]/var/www/html/venv/lib/python3.12/site-packages/torch/testing/_internal/common_distributed.pyr#   r#   7   s    NLr-   r#   backend_unavailableH   z5Skipped because distributed backend is not available.small_worldsizeI   z Skipped due to small world size.odd_worldsizeW   zSkipped due to odd world size.no_cudaJ   zCUDA is not available.zmulti-gpu-1K   zNeed at least 1 CUDA devicezmulti-gpu-2M   zNeed at least 2 CUDA deviceszmulti-gpu-3P   zNeed at least 3 CUDA deviceszmulti-gpu-4Q   zNeed at least 4 CUDA deviceszmulti-gpu-5R   zNeed at least 5 CUDA deviceszmulti-gpu-6S   zNeed at least 6 CUDA deviceszmulti-gpu-7T   zNeed at least 7 CUDA deviceszmulti-gpu-8U   zNeed at least 8 CUDA devicesncclL   z#c10d not compiled with NCCL support
skipIfRocmN   zTest skipped for ROCmno_peer_accessO   z'Test skipped because no GPU peer accessgenericV   zHTest skipped at subprocess level, look at subprocess log for skip reasonimporterrorX   z"Test skipped due to missing importc                       e Zd Zi Zh ded<    e       ed<   ddhed<   ddhed<   i Zh ded	<   h ded
<   h ded<   h ded<    e       ed<   y)DistTestCases>   mpiuccr?   allgather_coalescedr   r?   rL   zsendrecv anysourcezcpu barrier>   rL   gloor?   gpucudaddpsubgrouppluginN)r&   r'   r(   skip_collectivesetbackend_featurer,   r-   r.   rJ   rJ   U   sx     O-CO)* #OH-3UOO()&,e_OM" O4OE5OF4OE"9OJ #OHr-   rJ   c                 .     t                fd       }|S )zSkips if the world size exceeds the number of GPUs, ensuring that if the
    test is run, each rank has its own GPU via ``torch.cuda.device(rank)``.c                  h   t         j                  j                         s&t        j                  t
        d   j                         t        t        j                  d         }t         j                  j                         |k  r)t        j                  t
        d|    j                          | i |S )Nr5   
WORLD_SIZE
multi-gpu-)torchrP   is_availablesysexit
TEST_SKIPSr$   r)   osenvirondevice_count)argskwargs
world_sizefuncs      r.   wrapperzskip_if_no_gpu.<locals>.wrapperk   s    zz&&(HHZ	*445L12
::""$z1HHZ*ZL 9:DDET$V$$r-   r	   rf   rg   s   ` r.   skip_if_no_gpurj   g   s"     4[% % Nr-   c                 .     t                fd       }|S )Nc                      t         j                  d   dk7  rEt        t         j                  d         dk  r&t        j                  t
        d   j                          | i |S )NBACKENDrK   rY      r1   r`   ra   r)   r]   r^   r_   r$   rc   rd   rf   s     r.   rg   z(skip_if_small_worldsize.<locals>.wrappery   sR    JJy!U*BJJ|4L0MQR0RHHZ 12<<=T$V$$r-   rh   ri   s   ` r.   skip_if_small_worldsizerq   x        
4[% % Nr-   c                 .     t                fd       }|S )Nc                      t         j                  d   dk7  rHt        t         j                  d         dz  dk(  r&t        j                  t
        d   j                          | i |S )Nrm   rK   rY   rn      r3   ro   rp   s     r.   rg   z&skip_if_odd_worldsize.<locals>.wrapper   sW    JJy!U*BJJ|4L0MPQ0QUV0VHHZ0::;T$V$$r-   rh   ri   s   ` r.   skip_if_odd_worldsizerv      rr   r-   c                       fd}|S )Nc                 4     t                fd       }|S )Nc                      dk(  rKt         j                  j                         k  r*t        j                  t
        d    j                         y  | i |S Nr?   rZ   )r[   rP   rb   r]   r^   r_   r$   )rc   rd   backendrf   ns     r.   rg   zCrequire_n_gpus_for_nccl_backend.<locals>.decorator.<locals>.wrapper   sM    & UZZ%<%<%>%Bj$45??@T,V,,r-   rh   )rf   rg   r{   r|   s   ` r.   	decoratorz2require_n_gpus_for_nccl_backend.<locals>.decorator   s     	t	- 
	- r-   r,   )r|   r{   r}   s   `` r.   require_n_gpus_for_nccl_backendr~      s     r-   c                      d } | S )Nc                 .     t                fd       }|S )Nc                      	 ddl m}m}  | i |S # t        $ r) t	        j
                  t        d   j                         Y y w xY w)Nr   )AutoModelForMaskedLM
BertConfigrG   )transformersr   r   ImportErrorr]   r^   r_   r$   )rc   rd   r   r   rf   s       r.   rg   z?import_transformers_or_skip.<locals>.decorator.<locals>.wrapper   sE    >
 T,V,, >M2<<=>s    /AArh   ri   s   ` r.   r}   z.import_transformers_or_skip.<locals>.decorator   s     	t		> 
		> r-   r,   )r}   s    r.   import_transformers_or_skipr      s     r-   c                     t         j                  j                         xr! t         j                  j                         | k\  S N)r[   rP   r\   rb   )xs    r.   at_least_x_gpur      s,    ::""$G)@)@)Ba)GGr-   c                       fd}|S )Nc                 2     t                fd       }|S )Nc                      t         j                  j                         r)t         j                  j                         k\  r | i |S t	        j
                  t        d    j                         y )NrZ   r[   rP   r\   rb   r]   r^   r_   r$   )rc   rd   rf   r   s     r.   rg   z4skip_if_lt_x_gpu.<locals>.decorator.<locals>.wrapper   sU    zz&&(UZZ-D-D-F!-KT,V,,HHZ*QC 01;;<r-   rh   )rf   rg   r   s   ` r.   r}   z#skip_if_lt_x_gpu.<locals>.decorator   s     	t	= 
	=
 r-   r,   )r   r}   s   ` r.   skip_if_lt_x_gpur      s     r-   c                       fd}|S )Nc                 4     t                fd       }|S )Nc                      dk7  r | i |S t         j                  j                         r)t         j                  j                         k\  r | i |S t	        j
                  t        d    j                         y rz   r   )rc   rd   r{   rf   r   s     r.   rg   z9nccl_skip_if_lt_x_gpu.<locals>.decorator.<locals>.wrapper   sm    & T,V,,zz&&(UZZ-D-D-F!-KT,V,,HHZ*QC 01;;<r-   rh   )rf   rg   r{   r   s   ` r.   r}   z(nccl_skip_if_lt_x_gpu.<locals>.decorator   s     	t	= 
	= r-   r,   )r{   r   r}   s   `` r.   nccl_skip_if_lt_x_gpur      s    	 r-   c                     | j                         }d|v sJ d|v sJ d|v sJ |d   }|j                  d      dk(  r|n|j                  d      d   }||v sJ d| d|        y )	N	iteration	has_errorerrorz
Exception raised from r   zDid not find expected z in ddp logging data error: )_get_ddp_logging_datafindsplit)	model_DDP
err_substrddp_logging_datalogging_erractuals        r.   verify_ddp_error_loggedr      s     668********&&&&"7+K ??56"< 	89!<  	+R	x'CK=QRr-   c                 .     t                fd       }|S )aJ  
    Convenience decorator to set/unset TORCH_NCCL_BLOCKING_WAIT flag. Note that use of
    this decorator will override the setting of TORCH_NCCL_ASYNC_ERROR_HANDLING for
    the particular test. After the test, both TORCH_NCCL_BLOCKING_WAIT and
    TORCH_NCCL_ASYNC_ERROR_HANDLING will be restored to their original values.
    c                     	 t         j                  d   }t         j                  d= 	 t         j                  d   }dt         j                  d<   	  | i |}|||t         j                  d<   ||t         j                  d<   S S # t        $ r d }Y jw xY w# t        $ r d }Y gw xY w# dt         j                  d<   w xY w# ||t         j                  d<   ||t         j                  d<   w w xY w)NTORCH_NCCL_ASYNC_ERROR_HANDLINGTORCH_NCCL_BLOCKING_WAIT1)r`   ra   KeyError)rc   rd    cached_nccl_async_error_handlingcached_nccl_blocking_waitretrf   s        r.   rg   z(with_nccl_blocking_wait.<locals>.wrapper   s   	4AC1B, 

<=	9:<***;% 69BJJ12	S''C 0; 5 

5 )49R

56 51  	4/3,	4  	-(,%	- 69BJJ12 0; 5 

5 )49R

56 5s@   $B B 	B> BBB# B& "B##B& &B;>-C+rh   ri   s   ` r.   with_nccl_blocking_waitr      s%     4[ S  SD Nr-   c                       fd}|S )zK
    Runs a test for each distributed debug level specified in levels.
    c                 2     t                fd       }|S )Nc                     t         j                  j                  dd       }D ][  }|t         j                  d<   t        j                           | i |}t        j
                          |I|t         j                  d<   ] S )NTORCH_DISTRIBUTED_DEBUG)r`   ra   getc10dset_debug_level_from_envbarrier)rc   rd   	old_levelr!   r   rf   levelss        r.   rg   z:with_dist_debug_levels.<locals>.decorator.<locals>.wrapper  sx    

'@$GI F8=

45--/D+F+(<EBJJ89F Jr-   rh   )rf   rg   r   s   ` r.   r}   z)with_dist_debug_levels.<locals>.decorator  s     	t	 
	 r-   r,   )r   r}   s   ` r.   with_dist_debug_levelsr     s    
$ r-   c                  @    t        t        j                          d      S )Nz+c10d was not compiled with the Gloo backend)r   r   is_gloo_availabler,   r-   r.   requires_gloor   *  !    )""$$5 r-   c           	         t        j                         st        d      S t        t        j
                  j                  j                         | k  d|  dt        j
                  j                  j                          d|       S )N+c10d was not compiled with the NCCL backendz0Requires NCCL version greater than or equal to: z	, found: z
, reason: )r   is_nccl_availabler   r   r[   rP   r?   version)r   msgs     r.   requires_nccl_versionr   1  sv    !!#*9
 	
 .JJOO##%/>wiyQVQ[Q[Q`Q`QhQhQjPkkuvyuz{
 	
r-   c                  @    t        t        j                          d      S )Nr   )r   r   r   r,   r-   r.   requires_ncclr   =  r   r-   c                  @    t        t        j                          d      S )Nz*c10d was not compiled with the UCC backend)r   r   is_ucc_availabler,   r-   r.   requires_uccr   C  !    )!!##4 r-   c                  @    t        t        j                          d      S )Nz*c10d was not compiled with the MPI backend)r   r   is_mpi_availabler,   r-   r.   requires_mpir   I  r   r-   c                 <     d _         t                fd       }|S )zSkips a test for ROCmTc                  n    t         s | i |S t        j                  t        d   j                         y )NrA   )r   r]   r^   r_   r$   rp   s     r.   rg   z*skip_if_rocm_multiprocess.<locals>.wrapperT  s-    (((L)334r-   )skip_if_rocm_multiprocessr	   ri   s   ` r.   r   r   P  s(    %)D"
4[5 5
 Nr-   c                  <    t        t        j                  dk(  d      S )Nwin32z8This unit test case is not supported on Windows platform)r   r]   platformr,   r-   r.   skip_if_win32r   ]  s    )B r-   	localhostru   T   )minutesFc                     t               }|rEt        |t        d      z        }t        j                  j
                  j                  | ||||      S t        j                  | |||||      S )zL
    Creates a TCP store. Retries if the chosen port is already in use.
    ru   )milliseconds)wait_for_workers	use_libuv)r   r)   r   r[   classes	dist_c10dTCPStorer   )	addrre   	is_mastertimeoutr   	jit_classr   porttimeout_milliseconds	            r.   create_tcp_storer   d  so     D!'I1,E"EF}}&&//$
I/B
 	
 }}$
I@P\e
 	
r-   i  !DISTRIBUTED_TESTS_DEFAULT_TIMEOUT300test_ddp_uneven_inputsi     test_join_kwargsc                     t         j                  dk(  s|  t        j                  j	                  d      S t        j                  j	                  |       S )Nr   z	127.0.0.1)hostname	interface)r]   r   r   ProcessGroupGloocreate_devicer   s    r.   r   r     sG    
||w)"3$$22K2HH$$22Y2GGr-   returnc                 Z    t         j                  | j                  d      d   t              S N.r   )TIMEOUT_OVERRIDEr   r   TIMEOUT_DEFAULT)test_ids    r.   get_timeoutr     s#    c 22 6HHr-   c               #   N  K   t               t               }} t        j                  t        j                  }}	 | |ct        _        t        _        t        j                  t        j                  f ||ct        _        t        _        y # ||ct        _        t        _        w xY wwr   )r
   r]   stdoutstderr)new_outnew_errold_outold_errs       r.   captured_outputr     sl     z8:WGzz3::WG2!('
CJjj#**$$!('
CJ'
CJs   5B%9B	 1B%	B""B%rankre   
num_inputsc                    ddt         dt         dt         dt         fd}dt         fd}t        |d      t        |d	      t        |d
      t        |d      t        |d	      t        |d
      fD cg c]N  }t        |      D cg c]  } ||| z  |z   ||z         c}t        |      D cg c]  } ||||z         c}fP c}}S c c}w c c}w c c}}w )z
    Generate a number of basic test cases for sparse reduction.
    These cover tensors with a varying number of sparse dimensions and a varying
    number of dense dimensions. The only reduction operation we support is sum.
    ru   r   re   sparse_dims
dense_dimsc           	         t        j                  t        j                  | dz         d| dz   f      }|gt        |      D cg c]  }d c}z   }t        |dz
        D ]A  }t        j                  |t        j
                  d| dz         f      }|j                  |       C t        j                  | dz   gt        |      D cg c]  }d c}z         }t        j                  |||      S c c}w c c}w )Nru   rn   )	r[   reshapearangerangecatzerosappendonessparse_coo_tensor)r   re   r   r  indices_shapevaluess           r.   generatez,simple_sparse_reduce_tests.<locals>.generate  s     --TAX 6D1HF5+<=a=={Q' 	%Aii%++a*B CDGLL$	% TAXJU:5F)G!)GGH&&w>>  > *Hs   	C+	C0
c           
      |    t        t        j                  t        |      D cg c]  } | ||       c}      S c c}w r   )r   operatoraddr  )fnre   r   s      r.   compute_sumz/simple_sparse_reduce_tests.<locals>.compute_sum  s2    LLE*<MND2dJ/N
 	
Ns   9
)r   rn      )r  )ru   r   )r)   r   r  )r   re   r   r  r  r  is          r.   simple_sparse_reduce_testsr    s    
?s 
? 
?# 
?s 
?
C 
 H!,H!,H!,H+H+H+
 	 z* :$q(*z*AB @EZ?PQ![Z*45Q	
  Rs$   5CC C/CC
Cr{   c           
          t         j                  j                         }t        |      }d}| |kD  r|| z  }t        |       D ci c]  }|t	        |||z  |dz   |z          }}|S c c}w )zMultigpu tests are designed to simulate the multi nodes with multi
    GPUs on each node. Nccl backend requires equal #GPUs in each process.
    On a single node, all visible GPUs are evenly
    divided to subsets, each process only uses a subset.
    ru   )r[   rP   rb   r  list)re   r{   nGPUsvisible_devicesnGPUs_per_processr  rank_to_GPUs          r.   init_multigpu_helperr    s     JJ##%EElO E!Z/ z" 	
4$5 5QBS8STUUK  	s   A&tmp_dirinit_methodc                    t        j                         at        j                  t        j
                  d<   t	        j                  t        j                  j                  t        j                  d             t	        j                  t        j                  j                  t        j                  d             t        j                  j                  t        j                  d      }t	        j                  |       | | t        j
                  d<   y t        t        j                  j                  |d      z   t        j
                  d<   y )NTEMP_DIRr   test_dirinit_dirINIT_METHODshared_init_file)
tempfileTemporaryDirectoryr  namer`   ra   mkdirpathjoinr   )r   init_dir_paths     r.   initialize_temp_directoriesr.    s    ))+G$\\BJJzHHRWW\\',,	23HHRWW\\',,
34GGLLz:MHH]$/

=!$/"'',,-3
 %


=!r-   c                  :    t         t         j                          y y r   )r  cleanupr,   r-   r.   cleanup_temp_dirr1    s     r-      c            	       &    e Zd ZdZdZdefdZedefd       Z	d Z
ddededd	f fd
Zd fdZd fdZdefdZddZddZ G d de      Zedefd       Zededededd	fd       Zdedd	fdZddZddZddZddZedefd       Z xZS )MultiProcessTestCaser   
   r   c                      y)NFr,   selfs    r.   _should_stop_test_suitez,MultiProcessTestCase._should_stop_test_suite  s    r-   c                     t         S r   DEFAULT_WORLD_SIZEr7  s    r.   re   zMultiProcessTestCase.world_size      !!r-   c                 V    t              fd       }t        j                  ||       S )Nc                 j    | j                   | j                  k(  r| j                         y          y r   )r   MAIN_PROCESS_RANK_join_processesr8  r  s    r.   rg   z1MultiProcessTestCase.join_or_run.<locals>.wrapper  s(    yyD222$$R(r-   r	   types
MethodTyper8  r  rg   s    ` r.   join_or_runz MultiProcessTestCase.join_or_run  .    	r	 
	 ..r-   method_name
methodNameNc                     |dk7  r|}t         |   |       t        | |      }t        | || j	                  |             y NrunTestsuper__init__getattrsetattrrG  r8  rI  rJ  r  	__class__s       r.   rP  zMultiProcessTestCase.__init__'  B     "$K%T;'k4#3#3B#78r-   c                     t         |           g | _        g | _        | j                  | _        t        j                  d      j                  | _	        i | _
        y )NF)delete)rO  setUpskip_return_code_checks	processesr@  r   r'  NamedTemporaryFiler)  	file_namepid_to_piper8  rT  s    r.   rX  zMultiProcessTestCase.setUp0  sH    ')$**	!44EBGGr-   c                 r    t         |           | j                  D ]  }|j                           g | _        y r   )rO  tearDownrZ  	terminate)r8  prT  s     r.   r`  zMultiProcessTestCase.tearDown9  s3     	AKKM	 r-   c                 F    | j                         j                  d      d   S r   idr   r7  s    r.   _current_test_namez'MultiProcessTestCase._current_test_nameC  s    wwys#B''r-   c                    g | _         t        t        | j                              D ]  }t        j
                  j                         \  }} || j                  j                  dt        |      z   || j                         | j                  |fdt        | dd      i      }|j                          t        j                  d||j                          || j"                  |j                   <   | j                   j%                  |        y )Nzprocess fake_pgF)targetr)  rc   rd   zStarted process %s with pid %s)rZ  r  r)   re   r[   multiprocessingPiperT  _runr+   rf  r\  rQ  startloggerinfopidr]  r  )r8  procr   parent_conn
child_connprocesss         r.   _start_processesz%MultiProcessTestCase._start_processesG  s    #doo./ 	+D&+&;&;&@&@&B#K~~**#d)+D335t~~zRwtY>	G MMOKK8$L,7DW[[)NN!!'*	+r-   c                 x    t         j                  j                  d      j                  }| j	                  |       y )Nspawn)r[   rj  get_contextProcessru  )r8  rq  s     r.   _spawn_processesz%MultiProcessTestCase._spawn_processesX  s,    $$009AAd#r-   c                       e Zd ZdZy)MultiProcessTestCase.Eventru   N)r&   r'   r(   GET_TRACEBACKr,   r-   r.   Eventr|  \  s    r-   r~  r   c                    t         j                  d|       	 t        j                  j	                  | |g      }| |v r| j
                  rt         j                  d|       y | j                         }t         j                  d||       |t        j                  j                  k(  rt        j                  d      5 }t        j                  |       |j                          |j                  d       | j!                  |j#                                t         j                  d|       d d d        ||v ry # 1 sw Y   xY w)Nz*Starting event listener thread for rank %sz:Pipe closed for process %s, stopping event listener threadzReceived event %s on process %szr+)moder   zProcess %s sent traceback)rn  ro  rj  
connectionwaitclosedrecvr4  r~  r}  r'  r[  faulthandlerdump_tracebackflushseeksendread)parent_pipesignal_piper   ready_pipeseventtmp_files         r.   _event_listenerz$MultiProcessTestCase._event_listener_  s   @$G)4499;:TUKk)%%KKTVZ #((*=udK066DDD!44$? G8$33H= ( a(#((9$?FG k)5  G Gs   :A,D55D>	test_namer\  c                 T     | |      }||_         ||_        |j                  ||       y r   r   r\  run_testclsr   r  r\  r  rd   r8  s          r.   rl  zMultiProcessTestCase._run~  s'    9~	"i-r-   c           	         t         j                  j                  d      \  }}t        j                  t
        j                  ||| j                  fd      }|j                          t        j                  dk7  r2t        j                  dk7  rt         j                  j                  d       dt        j                  d<   	  t        | |              ||j;                  d        |J |j=                          |j?                          y # t         j"                  $ r[}t$        j'                  d	| j                  |t)        |             t        j*                  t,        d
   j.                         Y d }~d }~wt0        $ r}t$        j3                  dt5        j6                         | j                  t
        j8                         |j;                  t5        j6                                t        j*                  t
        j8                         Y d }~;d }~ww xY w# ||j;                  d        |J |j=                          |j?                          w xY w)NF)duplexT)ri  rc   daemonr   darwinr   TORCH_SHOW_CPP_STACKTRACESz4Process %s skipping test %s for following reason: %srE   z;Caught exception: 
%s exiting process %s with exit code: %s) r[   rj  rk  	threadingThreadr4  r  r   rm  r]   r   _C'_set_print_stack_traces_on_fatal_signalr`   ra   rQ  unittestSkipTestrn  ro  r+   r^   r_   r$   	Exceptionr   	traceback
format_excTEST_ERROR_EXIT_CODEr  r,  close)r8  r  r  signal_recv_pipesignal_send_pipeevent_listener_threadsees           r.   r  zMultiProcessTestCase.run_test  s   -2-B-B-G-Gu-G-U** ) 0 0'77/;!

 	##%<<7"s||x'? HH<<TB36

/0	 $GD)$&   + %%d+(444!&&(+    	6KKF		S\^abd^e HHZ	*4455 	@LL0$$&		3G3\3\ Y1134HH)>>??	@  + %%d+(444!&&(s>    D	 	HAE2-H 2H>BHH HH 9Ic                    g }t        | j                        D ]h  \  }}|j                  | j                  |j                     }	 |j                  t        j                  j                         |j                  ||f       j |D ]x  \  }}	 |j                  d      rK|j                  rt        j                  d|       ;|j!                         }t        j                  d||       nt        j                  d|       z y # t        $ r"}t        j                  d||       Y d }~d }~ww xY w# t        $ r!}t        j                  d||       Y d }~d }~ww xY w)NzBEncountered error while trying to get traceback for process %s: %sr   z5Pipe closed for process %s, cannot retrieve tracebackz)Process %s timed out with traceback: 

%sz6Could not retrieve traceback for timed out process: %s)	enumeraterZ  exitcoder]  rp  r  r4  r~  r}  r  ConnectionErrorrn  r   pollr  ro  r  )r8  pipesr  rt  piper  r   r  s           r.   _get_timedout_process_tracebackz4MultiProcessTestCase._get_timedout_process_traceback  s>   #DNN3 		JAw'''4II288FFGLL!T+		   	JD$99Q<{{SUY ! $		ILLEtY LLPRV	 ' LL\^_ab 0 # XZ^`a s6   <D3D/ >D/	D,
D''D,/	E8EEc                    t        | j                               }t        j                         }d}	 	 t        | j                        D ]w  \  }}|j
                  t        j                  k(  s$t        d| d|j
                   d       t        j                  j                         }|D ]  }|j                           d} n |rnt        d | j                  D              rntt        j                         |z
  }	|	|kD  rA| j                          t        d| d       | j                  D ]  }|j                           nt        j                  d	       #t        j                         |z
  }
|| j                   v r| j#                  |
       n| j%                  |
       | j&                  j)                         D ]  }|j+                           y # | j&                  j)                         D ]  }|j+                           w xY w)
NFTProcess z terminated with exit code z", terminating remaining processes.c              3   8   K   | ]  }|j                   d u  y wr   )r  ).0rb  s     r.   	<genexpr>z7MultiProcessTestCase._join_processes.<locals>.<genexpr>  s     F!qzz-Fs   zTiming out after z" seconds and killing subprocesses.g?)r   re  timer  rZ  r  r4  r  printr[   rj  active_childrenra  allr  sleeprY  _check_no_test_errors_check_return_codesr]  r  r  )r8  r  r   
start_timesubprocess_errorr  rb  r  acelapsedelapsed_timer  s               r.   rA  z$MultiProcessTestCase._join_processes  s   dggi(YY[
 *	'7 FQ zz%9%N%NN&qc)DQZZLPrs +0*?*?*O*O*Q"1 +BLLN++/( $Ft~~FF))+
2W$88:+G94VW "^^ &&

3= @  99;3LT111**<8((6 ((//1 

((//1 

s   9G +D2G 1G>c                     t        | j                        D ]I  \  }}|j                  t        d| d| d      | j	                  | j
                  |j                         K y)zV
        Checks that we didn't have any errors thrown in the child processes.
        Nr  z timed out after  seconds)r  rZ  r  RuntimeErrorassertNotEqualr  )r8  r  r  rb  s       r.   r  z*MultiProcessTestCase._check_no_test_errors  sh     dnn- 	GDAqzz!"qc!2<.I   9 91::F	Gr-   c                 8   | j                   st        j                  d       y| j                   d   }t        | j                         D cg c]&  \  }}|j                  t
        j                  k(  r||f( }}}|r[d}|D ]I  \  }}| j                  |j                     j                         }|d| dt
        j                   d| dz  }K t        |      t        | j                         D ]h  \  }}|j                  t        d| d	| d
      | j                  |j                  |j                  d| d|j                   d|j                          j t        j                         D ]q  }	|j                  |	j                  k(  st        r1t        j!                  d| j#                         |	j$                          yt'        j(                  |	j$                         | j                  |j                  dd|j                   d|j                          yc c}}w )z
        Checks that the return codes of all spawned processes match, and skips
        tests if they returned a return code indicating a skipping condition.
        z<Note: no subprocesses were spawned, test was likely skipped.Nr    r  z exited with error code z and exception:

 terminated or timed out after r  zExpect process z+ exit code to match Process 0 exit code of z
, but got )r   6Skipping %s on sandcastle for the following reason: %sz Expected zero exit code but got z
 for pid: )rZ  rn  warningr  r  r4  r  r]  rp  r  r  assertEqualr_   r  r$   r   ro  re  r%   r  r  )
r8  r  first_processr  rb  errored_processesr   rt  error_messageskips
             r.   r  z(MultiProcessTestCase._check_return_codes  s?    ~~NNYZq) "$..1
1zz1FFF F
 

 E/ 
7 $ 0 0 = B B Dqc!9:N:c:c9d e''4oR9 u%% dnn- 		DAqzz!"qc!@hW  

&&%aS(STaTjTjSkkuvw  wA  wA  vB  C  		 %%' 	:D%%7 
 KKPRVRYRYR[]a]i]i "++DLL99	: 	""2=3I3I2J*UbUfUfTgh 	 	
S
s   
+Hc                      | j                   dk(  S )Nr   r   r7  s    r.   r   zMultiProcessTestCase.is_masterP  s    yyA~r-   rM  rM  r   N)r&   r'   r(   r@  r  boolr9  propertyr)   re   rG  r+   rP  rX  r`  rf  ru  rz  r   r~  staticmethodr  classmethodrl  r  r  rA  r  r  r   __classcell__rT  s   @r.   r4  r4  	  s   
   "C " "/9C 9 9UY 9(C (+"$    < . . . .W[ . .) # ) t ) V#J.`	G>
@ 4  r-   r4  subtest_configtest_fntest_kwargsc                    t        |j                               }|D cg c]  }|d   	 }}|D cg c]  }|d   	 }}t        j                  | D ]  }	t	        t        ||	            }
 | j                  di |
5  t        j                  j                           ||i ||
 t        j                  j                          ddd       t        j                           yc c}w c c}w # 1 sw Y   *xY w)a\  
    Runs a test function given by ``test_fn`` as a subtest according to the
    configurations specified by ``subtest_config``. This amortizes the
    costly setup overhead (including process spawn and initializing the
    process group) over the subtests.

    Args:
        subtest_config (Dict[str, List[Any]]): A mapping from subtest
            keyword argument name to a list of its possible values.
        test_fn (Callable): A callable that runs the actual test.
        test_args: Positional arguments to pass to ``test_fn``.
        test_kwargs: Keyword arguments to pass to ``test_fn``.
    r   ru   Nr,   )r  items	itertoolsproductdictzipsubTestr[   _dynamoresetr   r   )cls_instr  r  	test_argsr  subtest_config_itemsitemsubtest_config_keyssubtest_config_valuesr  subtest_kwargss              r.   run_subtestsr  U  s    * 9=^=Q=Q=S8T:N%O$d1g%O%OBV-W$d1g-W-W##%:; c"5v>?X// 	"MM!Y@+@@MM!	" 	 &P-W	" 	"s   C C%:AC**C3	c                      t         t         S 	 t        j                  g dd      j                  dk(  a t         S # t        $ r
 da Y t         S w xY w)a   
    If shell command `fi_info -p efa -t FI_EP_RDM` returns exit code 0 then we assume that the machine has
    Libfabric EFA interfaces and EFA software components installed,
    see https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/efa-start.html.
    )fi_infoz-pefaz-t	FI_EP_RDMF)checkr   )EFA_PROBE_RESULT
subprocessrun
returncodeFileNotFoundErrorr,   r-   r.   has_efar  {  sZ     #!NNFeT__cdd 	
   ! !s   &: AAc                  "    t               rddgS dS )a  
    If the machine has Libfabric EFA interfaces and EFA software components installed it may cause
    'RuntimeError: In operator() at tensorpipe/common/ibv.h:172 "": Operation not supported' if tensorpipe
    uses InfiniBand transport, so we exclude it from tensorpipe transports,
    see https://github.com/pytorch/pytorch/issues/73885 and https://github.com/pytorch/pytorch/issues/65022
    shmuvN)r  r,   r-   r.   tp_transportsr     s     $IE4=/4/r-   c                 d      t        t        |      S d t                fd       }|S )z+
    Wrapper to use with a test method
    )r   re   c                      t               t        j                         }fd fd}g }t               D ]=  }t	        j
                  |||f      }|j                          |j                  |       ? |S )Nc                  >     t         j                  j                  k(  S r   r   distributed_c10d_worldworlds   r.   world_is_validzaspawn_threads_and_init_comms.<locals>._run_test_method_with_multi_threads.<locals>.world_is_valid      D118888r-   c                 ~   t        j                  d| |       	                  rt        j                          y y # t        $ rR}t        j                  j                  | t        j                         f       t        j                  |       Y d }~sd }~ww xY w#         rt        j                          w w xY w)Nthreadedr{   r   re   store)r   init_process_groupBaseExceptionMultiThreadedTestCaseexception_queueputr]   exc_infor    exception_handledestroy_process_group)r   world_pgr  excallbackr	  re   s       r.   workerzYspawn_threads_and_init_comms.<locals>._run_test_method_with_multi_threads.<locals>.worker  s    ##"*E1
 "#..0 $ ! 7%55994:PQ!222667
 "#..0 $s*   A   	B	ABB BB B<ri  rc   )r   r   	HashStorer  r  r  rm  r  )	re   r  global_storer  threadsr   tr  r	  s	   ``     @@r.   #_run_test_method_with_multi_threadszIspawn_threads_and_init_comms.<locals>._run_test_method_with_multi_threads  sq    $&~~'	9	1 *% 	D  dE<5PQAGGINN1	
 r-   c                 X    t         j                  j                  j                  d       	   fd      }t        j                  |       t         j                  j                  j                  d       y # t         j                  j                  j                  d       w xY w)NTc                       g i S r   r,   )rc   rf   rd   r8  s   r.   <lambda>z?spawn_threads_and_init_comms.<locals>.wrapper.<locals>.<lambda>  s    dSWNiZ^NibhNi r-   F)r[   r  _distributed_c10d_set_thread_isolation_moder  _join_threads)r8  rc   rd   r  r   rf   re   s   ``` r.   rg   z-spawn_threads_and_init_comms.<locals>.wrapper  sq     	""==dC	I9*FijG!//>HH&&AA%HEHH&&AA%Hs   %A> >+B))r   spawn_threads_and_init_commsr	   )rf   r   re   rg   r   s   ` ` @r.   r'  r'    sD     |('j
 	

< 4[I I Nr-   c                       e Zd ZdZ ej
                         ZdZd Zdde	de	ddf fdZ
d	 Zd
 Zd fdZ fdZd Zed        Zd Zed        Zed        Zedefd       Zede	fd       ZddddZddddZ xZS )r  a5  
    Test runner that runs all tests with the in-proc process group using
    multiple threads with the threaded process group.

    Each test spawns world_size threads and run the test method in each thread.

    Difference from regular MultiProcess test runner:
    Must explicitly defines SetUp and call self._spawn_threads() to run the tests.
    Cannot use setUp / tearDown (must use perThreadSetup / perThreadShutdown)
        to set up / tear down each thread when running each test.
    No global state possible
        How bad of a limitation is this?
    r   c                 V    t              fd       }t        j                  ||       S )Nc                     | j                   | j                  k(  r| j                  | j                         y          y r   )r   MAIN_THREAD_RANKr&  r  rB  s    r.   rg   z2MultiThreadedTestCase.join_or_run.<locals>.wrapper  s.    yyD111""4<<4r-   rC  rF  s    ` r.   rG  z!MultiThreadedTestCase.join_or_run  rH  r-   rI  rJ  r   Nc                     |dk7  r|}t         |   |       t        | |      }t        | || j	                  |             y rL  rN  rS  s       r.   rP  zMultiThreadedTestCase.__init__  rU  r-   c                      y r   r,   r7  s    r.   perThreadSetUpz$MultiThreadedTestCase.perThreadSetUp  s    r-   c                      y r   r,   r7  s    r.   perThreadTearDownz'MultiThreadedTestCase.perThreadTearDown  s    r-   c                 x    t         |           | j                  | _        g | _        dt
        j                  d<   y)z
        setUp only set up things in the main thread, if you want to configure things
        in the spawned threads, use perThreadSetUp
        r   r  N)rO  rX  r+  r   r  r`   ra   r^  s    r.   rX  zMultiThreadedTestCase.setUp  s1    
 	))	36

/0r-   c                 0    t         |           g | _        y)z
        tearDown only set up things in the main thread, if you want to configure things
        in the spawned threads, use perThreadTearDown
        N)rO  r`  r  r^  s    r.   r`  zMultiThreadedTestCase.tearDown  s    
 	r-   c                    t         j                  j                  j                  d       | j                  }t               t        j                         | j                  _	        fd} |       st        d      t        | j                        D ]e  }t        j                  | j                  j                  ||| j                  f      }|j!                          | j"                  j%                  |       g y)zk
        class method to spawn threads and run test, use this method in the SetUp of your TestCase
        Tc                  >     t         j                  j                  k(  S r   r  r  s   r.   r	  z<MultiThreadedTestCase._spawn_threads.<locals>.world_is_valid  r
  r-   zInvalid worldr  N)r[   r  r$  r%  rf  r   r   r  rT  r  r  r  re   r  r  rl  rm  r  r  )r8  r  r	  r   r  r  s        @r.   _spawn_threadsz$MultiThreadedTestCase._spawn_threads  s     	""==dC++	$&&*nn&6#	9 //$//* 	#D  (;(;9dTXTcTcBdeAGGILL"	#r-   c                     | |      }||_         t        |d      rWt        j                         |_        t
        j                  |j                  _        t
        j                  |j                  _	        |j                  |||       y )N_tls)r   hasattrr  localr7  r   
_precision	precision_rel_tolrel_tolrun_test_with_threaded_pg)r  r  r   re   rd   r8  s         r.   rl  zMultiThreadedTestCase._run$  sb    9~	 4 !)DI"*"5"5DII ( 1 1DII&&y$
Cr-   c                    t        j                  d||| j                  j                         | j	                          	  t        | |              t        j                          | j                          y# t        $ rN}| j                  j                  |t        j                         f       t        j                  |       Y d}~wd}~ww xY w# t        j                          | j                          w xY w)zd
        Run the current test associated with `test_name` using the threaded process group.
        r  r  N)r   r  rT  r  r.  rQ  r  r  r  r]   r  r    r  r  r0  )r8  r  r   re   r  s        r.   r>  z/MultiThreadedTestCase.run_test_with_threaded_pg4  s     	TjHcHc	
 		%$GD)$&
 &&(""$  	3  $$dCLLN%;<..r22	3 &&(""$s*   A5 5	C>ACC CC &C5c           
         t         }	 t        |      D ]f  \  }}|j                  t        d|             |j	                         s2t
        j                  j                  |t        t        d| d      d ff       h t        j                          g }| j                  j                         sF| j                  j                         }|j                  |       | j                  j                         sFt                t        j                   j"                  j%                  d       | j'                  |||       y # t                t        j                   j"                  j%                  d       w xY w)Nr   zRank failed to join in under r  F)r   r  r,  maxis_aliver  r  r  TimeoutErrorr    r  emptyr   r  r   r[   r  r$  r%  r  )r  r  r  r   idxthreadfailed_ranksfailures           r.   r&  z#MultiThreadedTestCase._join_threadsG  s-   !	I(1 VC7O,??$)99== , ,&CG9H$U!" !%	 ##%L))//1--113##G, ))//1 #$HH&&AA%Hgr: #$HH&&AA%Hs   <D9 B,D9 95E.c           	         d}d}|D ]1  \  }}|d   }t        |t        j                        r;t        j	                  d||t        |             |dk  sMt        d   j                  }at        |t              r)d| d| d	}	t        j                  |	       t        |	      t        |t              rEdj                  t        j                  |       }	t        j                  d
|	|       |d| d|	 dz  }t        |t              st!        |j"                        t$        k(  s|dk  s&|j"                  }4 t'        |      dkD  rt        |      |dkD  rqt        j)                         D ]Y  }
||
j                  k(  st*        r#t        j	                  d||
j,                          y t        j                  |
j,                         y y )Nr  r   ru   z3Thread %s skipping test %s for following reason: %sr   rE   zThread r  z	 seconds
z'Caught exception: 
%s exiting thread %sz exited with exception:
r  r  )
isinstancer  r  rn  ro  r+   r_   r$   rC  r   r  r  r,  r  format_exception
SystemExittypecoder)   lenr  r   r%   )r  rG  r   r  	error_msg	skip_coder   r  excr   r  s              r.   r  z)MultiThreadedTestCase._check_return_codese  s    		* 	)ND(1+C#x001I4QSUXY\U] q= *9 5 ? ?IC.v%DWIZXS!"3''C+ggi88(CD>T dV#<SED	 C,>S(Y] #I-	)2 y>Ay))q="))+ 	>.$TVXZ^ZfZf &//==	> r-   c                     t         S r   r;  r7  s    r.   re   z MultiThreadedTestCase.world_size  r=  r-   c                 F    | j                         j                  d      d   S r   rd  r7  s    r.   rf  z(MultiThreadedTestCase._current_test_name  s     wwys#B''r-   r   r  c                J    | j                   |k(  r| j                  |||       yy)z
        The reason why we have this util function instead of
        self.assertEqual is all threads are sharing one CPU RNG
        so the assertion result is only reliable on rank 0
        N)r   r  r8  r   yr   r   s        r.   assertEqualOnRankz'MultiThreadedTestCase.assertEqualOnRank  s'     99Q3' r-   c                H    | j                   |k(  r| j                  ||       y y r   )r   r  rV  s        r.   assertNotEqualOnRankz*MultiThreadedTestCase.assertNotEqualOnRank  s#    991% r-   r  r  r   )r&   r'   r(   __doc__queueQueuer  r+  rG  r+   rP  r.  r0  rX  r`  r5  r  rl  r>  r&  r  r  r)   re   rf  rX  rZ  r  r  s   @r.   r  r    s     "ekkmO/9C 9 9UY 9	7#* D D%& ; ;: .> .>` "C " " (C ( (( (&1 & &r-   r  c                        e Zd Zdeej
                  ej                  f   deddf fdZ	dej                  dej                  fdZ
 xZS )SaveForwardInputsModuleforward_inputscast_forward_inputsr   Nc                 t    t         |           t        j                  dd      | _        || _        || _        y )Nd   )rO  rP  nnLinearlr`  ra  r8  r`  ra  rT  s      r.   rP  z SaveForwardInputsModule.__init__  s2    
 	3$,#6 r-   r   c                     || j                   | <   | j                  | j                  r3|j                  | j                  j                  j
                              S |      S r   )r`  rf  ra  toweightdtyper8  r   s     r.   forwardzSaveForwardInputsModule.forward  sI    $%D!vv43K3Kadd466==../SSQRSSr-   r&   r'   r(   r   rd  Moduler[   Tensorr  rP  rm  r  r  s   @r.   r_  r_    sT    7RYY457 "7 
	7T T%,, Tr-   r_  c                        e Zd Zdeej
                  ej                  f   deddf fdZ	dej                  dej                  fdZ
 xZS )SaveForwardInputsModelr`  ra  r   Nc                 t    t         |           t        ||      | _        t        ||      | _        || _        y r   )rO  rP  r_  c1c2r`  rg  s      r.   rP  zSaveForwardInputsModel.__init__  s6    
 	).:MN).:MN,r-   r   c                 `    || j                   | <   | j                  | j                  |            S r   )r`  ru  rt  rl  s     r.   rm  zSaveForwardInputsModel.forward  s)    $%D!wwtwwqz""r-   rn  r  s   @r.   rr  rr    sQ    -RYY45- "- 
	-# #%,, #r-   rr  c              #     K   |st         j                  j                  |        dt        j                  d<   dt        j                  d<   |rp|rVt         j
                  j                  j                  j                  j                         }t        j                  d|| |       nt        j                  d| |       t         j                  j                          t         j                  j                  j                  j!                          	 d  t         j                  j                          t         j                  j                  j                  j!                          |rt        j"                          y y # t         j                  j                          t         j                  j                  j                  j!                          |rt        j"                          w w xY ww)	Nr   MASTER_ADDR6789MASTER_PORTfake)r{   re   r   r  r?   r   re   )r[   rP   
set_devicer`   ra   testing	_internaldistributedrh  	FakeStorer   r  r  r  utilscountersclearr  )r   re   init_pgrh  r  s        r.   _dynamo_dist_per_rank_initr    sH     

d# +BJJ} &BJJ}MM++77??IIKE##%	 ##F*M	MM	MM  &&()$$**,&&(  	$$**,&&( s    D
G%E9 A(G%9A)G""G%c                   @     e Zd ZdZe fd       Ze fd       Z xZS )#DynamoDistributedSingleProcTestCasez
    Test harness for single-process dynamo distributed tests,
    initializes dist process group.

    Prefer this for simple tests, as it's easier to debug.
    c                 `   t         |           | j                  j                  t	        j
                  t        j                  ddd             d| _        d| j                   | _	        d| j                  v rd n| j                  g| _
        t        j                  d| j                  d	       y )
Nr   12355)rx  rz  r   zcuda:rP   r?   ru   r|  )rO  
setUpClass_exit_stackenter_contextr   r  r`   ra   r   device
device_idsr   r  r  rT  s    r.   r  z.DynamoDistributedSingleProcTestCase.setUpClass  s    %%JJ

#.#*	
 SXXJ'
!'3::!5CHH:SXX!Dr-   c                 J    t        j                          t        |           y r   )r   r  rO  tearDownClassr  s    r.   r  z1DynamoDistributedSingleProcTestCase.tearDownClass  s    ""$r-   )r&   r'   r(   r[  r  r  r  r  r  s   @r.   r  r    s2     E E"    r-   r  c            	       d     e Zd ZdZ fdZ fdZedefd       Ze	dede
de
dd	fd
       Z xZS )"DynamoDistributedMultiProcTestCasea   
    Use this for tests that actually run on multiple GPUs.

    Decorate tests with @skip_if_lt_x_gpu(ngpu)

    Note: MultiProcTestCase spawns processes per test and is slow.
    Prefer MultiThreadedTestCase for most tests. Perhaps use this one
    sparingly for integration tests.
    c                 B    t         |           | j                          y r   )rO  rX  rz  r^  s    r.   rX  z(DynamoDistributedMultiProcTestCase.setUp  s    r-   c                     t         |           	 t        j                  | j                         y # t
        $ r Y y w xY wr   )rO  r`  r`   remover\  OSErrorr^  s    r.   r`  z+DynamoDistributedMultiProcTestCase.tearDown  s5    	IIdnn% 		s   1 	==r   c                 >    t         j                  j                         S r   )r[   rP   rb   r7  s    r.   re   z-DynamoDistributedMultiProcTestCase.world_size  s    zz&&((r-   r   r  r\  Nc                 T     | |      }||_         ||_        |j                  ||       y r   r  r  s          r.   rl  z'DynamoDistributedMultiProcTestCase._run#  s)     9~	"i-r-   )r&   r'   r(   r[  rX  r`  r  r)   re   r  r+   rl  r  r  s   @r.   r  r  
  s^      )C ) ) . . . .W[ . .r-   r  c            	            e Zd ZU dZeed<   dZeed<   dZee	   ed<   e
ej                  de	fd              Ze
dd	       Ze
 fd
       Ze
 fd       Ze
	 ddededee	   fd       Z xZS )MultiProcContinousTestrn   re   r   r   N	rdvz_filer   c                     t        d      )z
        ProcessGroup backend str.
        To be customized by sub test classes, e.g. "nccl".
        Here we raise error.
        z/Please implement backend_str in your test class)NotImplementedError)r  s    r.   backend_strz"MultiProcContinousTest.backend_str5  s     ""STTr-   c                      y)z
        ProcessGroup init options.
        To be customized by sub test classes, e.g. ProcessGroupNCCLOpTest
        Here we return None.
        Nr,   )r  high_priority_streams     r.   optszMultiProcContinousTest.opts?  s     r-   c                 P   t         |           d| j                  cxk  r| j                  k  s'n t	        d| j                   d| j                         | j
                  r+t        j                  | j
                  | j                        }nd}| j                         }| j                         }t        d|       t        j                  || j                  | j                  ||       t        j                  j                         | _        t        d| j                   d       y)	z
        Class-scope test fixture. Run once for entire test class, before any test starts.
        Set up the process group.
        r   zBRank must be set and in the range of 0 to world_size. World size: z Rank: NzTesting backend=)r{   re   r   r  
pg_optionsRank z setup complete)rO  r  r   re   r  r  r   	FileStorer  r  r  r  r  _get_default_grouppg)r  r  r  r{   rT  s       r.   r  z!MultiProcContinousTest.setUpClassH  s     	CHH-s~~-"~~.gchhZA  ==NN3==#..AE Exxz//#!
#$~~	
 &&99;chhZ/0r-   c                     t        j                          t        |           | j                  r 	 t        j                  | j                         t        d| j                   d       y# t        $ r Y %w xY w)z
        Class-scope test fixture. Run once for entire test class, after all tests finish.
        Tear down the process group.
        r  z teardown completeN)
r   r  rO  r  r  r`   r  r  r  r   r  s    r.   r  z$MultiProcContinousTest.tearDownClassg  sd     	""$==		#--( 	chhZ123  s   A* *	A65A6c                 B    || _         || _        || _        t                y)ad  
        This is an entry point for each rank to run the tests in `MultiProcContinousTest`.
        In this entry point, we set the class variables for the test class.
        Then we run all tests.

        Note:
        - This helper only works for a subclass of `MultiProcContinousTest`.

        Example:
        - See `test_c10d_ops_nccl.py`.
        N)r   re   r  r   )r  r   re   r  s       r.   run_rankzMultiProcContinousTest.run_rankw  s    & #!r-   )Fr   )r&   r'   r(   re   r)   r*   r   r  r   r+   r  abcabstractmethodr  r  r  r  r  r  r  s   @r.   r  r  ,  s     JD#N#Ix}#UC U  U   1 1< 4 4 
 $(	  C=	 r-   r  r   )ru   r  )TF)|r  r  r  loggingrj  r`   r\  r  r]   r'  r  r  r  rD  r  
contextlibr   dataclassesr   datetimer   enumr   	functoolsr   r   r	   ior
   typingr   r   r   r   r   r   r   r   unittest.mockr   r[   torch._dynamo.test_casetorch.cuda.nccltorch.distributedr  r   torch.nnrd  $torch.testing._internal.common_utilsr   r   r   r   r   r   r   r   r   r   5torch.testing._internal.distributed.multi_threaded_pgr   r   r    r  basicConfigINFO	getLoggerr&   rn  r#   r_   rJ   rj   rq   rv   r~   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r)   getenvr   r   r   r   r  r+   r  r  r(  r*   r.  r1  r<  r4  r  r  r  r  r   r'  r  ro  r_  rr  r  r  	test_caser  r  r  r,   r-   r.   <module>r     sk        	   
       % !   , ,  P P P          
    ',, '			8	$z 
8
C x$FG	
 Xb"BC x45 8B => 8B >? 8B >? 8B >? 8B >? 8B >? 8B >? 8B >? HR>?  (267!" hr#LM#$ x
V%* 8B DE+
2 & & &""&H
R$+\4	

 	a 
 
0 O)"))$GOPO,c2  +.'(HIC I 2 2(S (c (s (XS 3 * 26(--	. 5
Xc] 
d 
"  I8 IX
d3i( 
 F   &0 
3E5pZ&H Z&zTbii T #RYY # ) ): %--*A*A*J*J  @.)= .DbX br-   