
    sgC                     H   d dl Z d dlmZmZmZmZmZmZmZm	Z	 d dl
Z
d dlmZ d dlmZ d dlmZ ddgZde	e
j$                  ee   eed	f   eeef   f   d
ee
j$                     fdZ	 	 ddee   dee   deeeeef         deeee	ee
j,                  f            d
ee   f
dZy)    N)AnycastDictListOptionalSequenceTupleUnion)ExceptionWrapper)_get_device_index)Module	get_a_varparallel_applyobj.returnc                 p   t        | t        j                        r| S t        | t        t        f      r3t        t        |       D ]   }t        |t        j                        s|c S  t        | t              rAt        t        | j                               D ]   }t        |t        j                        s|c S  y )N)	
isinstancetorchTensorlisttuplemapr   dictitems)r   results     S/var/www/html/venv/lib/python3.12/site-packages/torch/nn/parallel/parallel_apply.pyr   r      s     #u||$
#e}%)S) 	F&%,,/	 #t)SYY[1 	F&%,,/	     modulesinputs
kwargs_tupdevicesc                 r   t        |       t        |      k(  sJ dt        |        dt        |              |t        |       t        |      k(  s,J t        t        t        t        f   i       ft        |       z  }|t        |       t        |      k(  sJ dgt        |       z  }|D cg c]  }t        |d       }}|D cg c]!  }t        j                  j                  |      # }}t        j                         i t        j                         t        j                         c	 	 ddt        dt        dt        dt        t        t        f   d	t        t         t        t        j"                  f      d
t        t        j                  j$                     ddffd}t        |       dkD  rt'        t)        | ||||            D 	
cg c](  \  }\  }}	}
}}t        j*                  ||||	|
||f      * }}}
}	}}}|D ]  }|j-                           |D ]  }|j/                           n |d| d   |d   |d   |d   |d          g }t1        t        |            D ]8  }|   }t3        |t4              r|j7                          |j9                  |       : |S c c}w c c}w c c}}}
}	}}w )a  Apply each `module` in :attr:`modules` in parallel on each of :attr:`devices`.

    Args:
        modules (Module): modules to be parallelized
        inputs (tensor): inputs to the modules
        devices (list of int or torch.device): CUDA devices

    :attr:`modules`, :attr:`inputs`, :attr:`kwargs_tup` (if given), and
    :attr:`devices` (if given) should all have same length. Moreover, each
    element of :attr:`inputs` can either be a single object as the only argument
    to a module, or a collection of positional arguments.
    zThe number of modules z& is not equal to the number of inputs NTimoduleinputkwargsdevicestreamr   c                 6   t        j                  	       |<t        |      }|
5  t        d|  d      | <   d d d        y |j	                         }|t         j
                  j                  |      }	 t         j
                  j                  |      5  t         j
                  j                  |      5  t         j                  j                  d      5  t        |t        t        f      s|f} ||i |}d d d        d d d        d d d        
5  | <   d d d        y # 1 sw Y   y xY w# 1 sw Y   6xY w# 1 sw Y   :xY w# 1 sw Y   >xY w# 1 sw Y   y xY w# t        $ r0 
5  t        d|  d|       | <   d d d        Y y # 1 sw Y   Y y xY ww xY w)Nzin replica zQ, no device was provided and no tensor input was found; device cannot be resolved)wherecuda)enabledz on device )r   set_grad_enabledr   r   
get_devicer+   current_streamr'   r(   ampautocastr   r   r   	Exception)r#   r$   r%   r&   r'   r(   toutputautocast_enabledgrad_enabledlockresultss           r   _workerzparallel_apply.<locals>._workerD   s    	|,>% Ay !1 +A3 /4 4"GAJ
 \\^F>ZZ..v6F	""6* 2EJJ,=,=- 2yy!!&2B!C2 "%$7"HE1&12 2 2  $#
$ $#
 
2 2 2 2 2 2$ $ 	 -'s+fX>
  	s   D#7E  E6"D;"D/:D;E

E EE #D,/D84D;;E	 EEE EE E F*F
 F
F	FF   )targetargsr   NN)lenr   r   strr   r   r   r+   r/   	threadingLockis_grad_enabledis_autocast_enabledintr   r   r
   r'   Stream	enumeratezipThreadstartjoinranger   r   reraiseappend)r   r   r    r!   xstreamsr9   r#   r$   r%   r&   r'   r(   threadsthreadoutputsr4   r5   r6   r7   r8   s                    @@@@r   r   r      s   $ w<3  b	G~-STWX^T_S`ab  7|s:...4S>2.03w<?
7|s7|+++&3w<'3:;a D);G;5<=uzz((+=G=>>DG!!# #L" 6:.2### # S#X	#
 sELL012# **+# 
# #J 7|a
 ?HGVZ'B?	
 
 ;:FE666 a%O
 
  	FLLN	 	FKKM	 	71:vay*Q-WQZPG3v; f./NNv	
 NK <=\
s   .J%&J*-J/r=   )r@   typingr   r   r   r   r   r   r	   r
   r   torch._utilsr   torch.cuda._utilsr   torch.nn.modulesr   __all__r   r   r?   rD   r'   r    r   r   <module>rY      s     J J J  ) / # (
)	u||T#Yc3hc3hG	Hell( 6:FJ	bfbSMb $sCx.12b hxc5<<.?(@ABC	b
 
#Ybr   