
    sg_                     d   d dl Z d dlmZmZmZmZmZmZ d dlm	Z	 d dl
Z
d dlZd dlmZ d dlmZ d dlmZ d dlmZ dd	gZ e
j*                  e      xZZ ed
       G d d             Z ed
      	 	 	 ddedej4                  j6                  deegef   deeeef      dee   dee   fd       Zy)    N)AnyCallableDictListOptionalSet)OrderedDict)compatibility)GraphModule)Node)lazy_format_graph_code	Partitionsplit_moduleT)is_backward_compatiblec                   $    e Zd ZdefdZdefdZy)r   namec                     || _         d| | _        g | _        i | _        i | _        i | _        i | _        t        j                  j                  j                         | _	        i | _        i | _        y )Nsubmod_)r   submod_name
node_namesinputsoutputsdependencies
dependentstorchfxgraphGraphenvironmenttargets)selfr   s     O/var/www/html/venv/lib/python3.12/site-packages/torch/fx/passes/split_module.py__init__zPartition.__init__   sc    	$TF+%'')(*-/+-+088>>+?+?+A
-/')    returnc                     d| j                    d| j                   d| j                   d| j                   d| j                   d| j
                   S )Nzname: z
,
 nodes: z,
 inputs: z,
 outputs: z,
 partitions depended on: z,
 partition dependents: )r   r   r   r   r   r   )r!   s    r"   __repr__zPartition.__repr__   sb    TYYK  ' (} % '((,(9(9': ;&&*oo%68	
r$   N)__name__
__module____qualname__strr#   r'    r$   r"   r   r      s    
*S 
*
# 
r$   mroot_msplit_callbackqualname_mapkeep_original_orderkeep_original_node_namec                    0=>?@ABCD t         j                  dt        d d             dt        dt        t
        t        f   dt        t
        t        j                  j                  j                  f   f> fd}d	d
l
}i Bi @i Ddt        dt        t           fBDfdCBfd}t        j                  j                  t        j                  j                  t        j                  j                   g}	t#               }
t#               }i }d
}t%               } j&                  j(                  D ]x  00j*                  j-                  d      x}ct/        |t        j0                        rIt/        |j2                  j4                  x}|j6                        r|Dvr0D|j2                  j4                  <   0j8                  dv r |0       0j8                  dk(  r0j:                  |	v r0j:                  t        j                  j                   k(  rRt=        0j>                        dk(  sJ t/        0j>                  d	   t@              sJ 0}t%         0      h      |
|<   n	0j:                  t        j                  j                  k(  rJtC        d 0j>                  D              sJ |jE                  0       t%         0      h      |0<   d
|0<   n0j:                  t        j                  j                  k(  rqt=        0j>                        dk(  sJ |0j>                  d	      jE                   0             |jG                  0j>                  d	          0|0j>                  d	   <   ||
|   jE                   0             |D ]  }||   jE                   0              { tC        d |jI                         D              sJ d       |jK                         D ci c]  \  }}|tM        |       }}}|
jK                         D ci c]  \  }}|tM        |       }
}}tN        jQ                  tR        jT                        r,tN        j                  d|       tN        j                  d|
       tA        |      xs tA        |
      }d} j&                  j(                  D ]  00@0jV                  <   0j8                  dv r"0j8                  dk(  r;t        j                  j&                  jY                  0j>                  d	   Cfd       l|r 0      }||k  sJ d| d|        |}0j:                  |	vst        j                  j&                  jY                  0j>                  0Cfd       t        j                  j&                  jY                  0jZ                  0Cfd        t]        Bj_                               }g }BjK                         D ],  \  }At=        Aj`                        r|jc                  |       . g }|rw|je                         }|jc                  |       B|   jf                  D ]A  }B|   j`                  je                  |       B|   j`                  r1|jc                  |       C |rwt=        |      t=        B      k7  rti        d       ||
fD ]  }|jK                         D ]  \  0}t=        |      d	kD  sJ 0Bt        |d	            jj                  0<   |dd
 D ]  }Bt        |         AAj&                  jm                  0j8                  0j:                  to        d! 0j>                  D              i 0jp                  "      } 0j*                  js                         | _        | Ajj                  0<      |D ]v  }B|   AAjt                  D ]`  }!Aj&                  jw                  |!@|!   jp                  #      }"@|!   j*                  js                         |"_        |"Ajj                  @|!   <   b x  j&                  j(                  D ]  0ty        0d$      sB0jz                     AAjj                  ?t        j                  j&                  jY                  0j>                  ?fd%      }#t        j                  j&                  jY                  0jZ                  ?fd&      }$0j8                  d'vr0j:                  }%n0j:                  j}                  d(      }& }'|&D ]3  }(ty        |'|(      st        d)0j:                   d*      t        |'|(      }'5 d+j                  |&      }%|'Aj                  |%<   | Aj                   d(|% })0j:                  ||)<   t/        |#tn              sJ t/        |$t              sJ r0jV                  nd
}*Aj&                  jm                  0j8                  |%|#|$0jp                  |*,      } 0j*                  js                         | _        | Ajj                  0<    |fD ]  }t        |      D ]  0|0   }t=        |      d	kD  sJ |d
d D ]  }Bt        |         A|0   }+|+J d-       Aj&                  jm                  |+j8                  |+j:                  Ajj                  0   fi |+jp                  "      } |+j*                  js                         | _           i },i =t        j                  j&                  j                         >i }-|s) j&                  j(                  D ]  0 |0=|-      \  =}- n* j&                  j(                  D ]  00|,0jV                  <    |s|n|}.t%               }/ j&                  j(                  D 0cg c]  }0|0j8                  d.k(  s|0 }1}0|.D ]  }B|   Ato        @Afd/Aj                  D              }2t=        |2      }3|3dk(  rAj&                  j                  |2d	          n |3dkD  rAj&                  j                  |2       |ruAjt                  D 4cg c]  }4|4|1vs|,|4    }5}4|1D ]%  00|/v r |0=|-      \  =}6|/jE                  0       ' |5D ]%  00|/v r |0=|-      \  =}-|/jE                  0       ' t        j                  j                  j                  Aj                  Aj&                        |-Aj                  <   >j                  Aj                  to        =fd0Ajt                  D                    }7t=        Aj                        }8|8dkD  rZt        j                  j                  j                  |7      }9t        Aj                        D ]  \  }:};|9|:   j2                  =|;<    |8dk(  s|7=t        t        Aj                              <     j&                  j(                  D ][  00j8                  dk(  s>j                  t        j                  j&                  jY                  0j>                  d	   =fd1             ] t        j                  j                  j                  |->      }<t         j                  dt        d2|<d             |<S c c}}w c c}}w c c}0w c c}4w )3a  
    Creates subgraphs out of main graph

    Args:
        m (GraphModule): Graph module to split
        root_m (torch.nn.Module): root nn module. Not currently used. Included
            because the root nn module is usually transformed via
            torch.fx._symbolic_trace.symbolic_trace (see example below)
        split_callback (Callable[[Node], int]): Callable function
            that maps a given Node instance to a numeric partition identifier.
            split_module will use this function as the policy for which operations
            appear in which partitions in the output Module.
        qualname_map: Optional[Dict[str, str]]: optional output parameter that returns a
            mapping from new target names in the module after split to old target
            names in the original module.
        keep_original_order: Optional[bool]: keep the original order of the GraphModule
            or use the Topological order of the new constructed GraphModule


    Returns:
        GraphModule: the module after split.

    Example:

        This is a sample setup:

            import torch
            from torch.fx.symbolic_trace import symbolic_trace
            from torch.fx.graph_module import GraphModule
            from torch.fx.node import Node
            from torch.fx.passes.split_module import split_module

            class MyModule(torch.nn.Module):
                def __init__(self) -> None:
                    super().__init__()
                    self.param = torch.nn.Parameter(torch.rand(3, 4))
                    self.linear = torch.nn.Linear(4, 5)

                def forward(self, x, y):
                    z = self.linear(x + self.param).clamp(min=0.0, max=1.0)
                    w = self.linear(y).clamp(min=0.0, max=1.0)
                    return z + w

            # symbolically trace model
            my_module = MyModule()
            my_module_traced = symbolic_trace(my_module)

            # random mod partitioning
            partition_counter = 0
            NPARTITIONS = 3

            def mod_partition(node: Node):
                global partition_counter
                partition = partition_counter % NPARTITIONS
                partition_counter = (partition_counter + 1) % NPARTITIONS
                return partition

            # split module in module with submodules
            module_with_submodules = split_module(
                my_module_traced, my_module, mod_partition
            )

        Output looks like this. Original graph is broken into partitions

            > print(module_with_submodules)
            GraphModule(
                (submod_0): GraphModule(
                    (linear): Linear(in_features=4, out_features=5, bias=True)
                )
                (submod_1): GraphModule(
                    (linear): Linear(in_features=4, out_features=5, bias=True)
                )
                (submod_2): GraphModule()
            )

            def forward(self, x, y):
                param = self.param
                submod_0 = self.submod_0(x, param, y);  x = param = y = None
                getitem = submod_0[0]
                getitem_1 = submod_0[1];  submod_0 = None
                submod_1 = self.submod_1(getitem, getitem_1);  getitem = getitem_1 = None
                getitem_2 = submod_1[0]
                getitem_3 = submod_1[1];  submod_1 = None
                submod_2 = self.submod_2(getitem_2, getitem_3);  getitem_2 = getitem_3 = None
                return submod_2

        Output of split module is the same as output of input traced module.
        This is an example within a test setting:

            > orig_out = my_module_traced(x, y)
            > submodules_out = module_with_submodules(x, y)
            > self.assertEqual(orig_out, submodules_out)
            True
    z%szpre split_moduleT)colorednodebase_mod_envbase_mod_attrsc                    | j                   dk(  r t        | j                        dkD  r| j                  d   nt        j                  j
                  }rX|t        j                  j
                  u rdn|f}j                  d| j                  || j                        || j                  <   n5j                  | j                  | j                  |      || j                  <   | j                  j                         || j                     _        ||fS | j                   dk(  rj                  | j                        || j                  <   | j                  j                         || j                     _        	}| j                  j                  d      D ]3  }t        ||      st!        d| j                   d	      t#        ||      }5 ||| j                  <   ||fS )
Nplaceholderr   r,   )args	type_expr)r;   default_valueget_attr.zNode target  not found!)oplenr:   inspect	Signatureemptycreate_noder   typer9   targetmetacopyr=   splithasattrAttributeErrorgetattr)
r5   r6   r7   r<   r:   attr_valatombase_mod_graphr2   r-   s
          r"   construct_graphz%split_module.<locals>.construct_graph   s   
 77m# #DII 2		!8I8I8O8O  '*g.?.?.E.EErMK[*8*D*D]TXT]T]dhtxt}t}*D*~TYY'*8*D*DKK499M +E +TYY' ,099>>+;L#( ^++ WW
"&4&=&=dkk&JL#+/99>>+;L#(H))#. 3x.(<}K)PQQ"8T23 +3N4;;'^++r$   r   Ndef_nodeuse_nodec                 v   ddl m} t        | dd       }t        |dd       }t        j	                  d| j
                  |||j
                  nd|       ||k7  ra|G|   }|j                  j                  | j
                         ||j                  j                  |       ||   }|j                  j                  | j
                         | j                  j                  d      x}t         ||      t              D ]  }|   }	|j                  j                  |	j
                         |   j                  dk7  s@t        |	dd       }
|
P|
   }|j                  j                  |	j
                         |j                  j                  |        ||j                  j                  |       y y y y )	Nr   )free_symbols_fx_partitionz*record_cross_partition_use %s (%s) %s (%s)-example_value)keyr9   )%torch.fx.experimental.symbolic_shapesrU   rM   logdebugr   r   
setdefaultr   r   rH   getsortedr+   r@   r   )rR   rS   rU   defineduseddef_partitionuse_partitiondef_valss_node	s_defineds_def_partition
partitionssymbol_to_nodes               r"   record_cross_partition_usez0split_module.<locals>.record_cross_partition_use   s    	G(OT:x$7		8MM7X5IHMMsTX	

 d?" *7 3%%00?#!,,77= *4 0$$//>  (}}00AAGN#L$9sC L!/!2%,,77D)!,//=@ )0(NI(42<Y2G / 7 7 B B6;; O / : : E Ed K!L" &!..99'B '1   r$   c                    t         |             }t        j                  d| j                  |       j	                  |      }|t        |      x|<   }|j                  j                  | j                         || _        y )Nz*instantiate_node_partition_mapping %s (%s))	r+   r[   r\   r   r^   r   r   appendrV   )r5   partition_name	partitionri   r/   s      r"   "instantiate_node_partition_mappingz8split_module.<locals>.instantiate_node_partition_mapping   sq    ^D12		>		>Z NN>2	5>~5NNJ~&##DII.+r$   rX   )r9   r=   outputcall_function   c              3   >   K   | ]  }t        |t                 y wN)
isinstancer   .0args     r"   	<genexpr>zsplit_module.<locals>.<genexpr>/  s     Jz#t44Js   c              3   $   K   | ]  }|d u 
 y wru   r,   )rx   vs     r"   rz   zsplit_module.<locals>.<genexpr>?  s     >q}>s   zautocast must exitzautocast_regions: %szgrad_regions: %s)r9   r=   rq   c                      | d       S ru   r,   )nrk   s    r"   <lambda>zsplit_module.<locals>.<lambda>U  s    (B1d(K r$   zRautocast or set_grad_enabled require monotonically increasing partitions:highest: z, this node's: c                      |       S ru   r,   rR   r5   rk   s    r"   r   zsplit_module.<locals>.<lambda>d  s    ,FxQU,V r$   c                      |       S ru   r,   r   s    r"   r   zsplit_module.<locals>.<lambda>g  s    .HSW.X r$   z cycle exists between partitions!c              3       K   | ]  }|  y wru   r,   rw   s     r"   rz   zsplit_module.<locals>.<genexpr>  s     8ss8s   )r@   rG   r:   kwargsr;   )r;   rV   c                     |    S ru   r,   r   r   s    r"   r   zsplit_module.<locals>.<lambda>  s    TU r$   c                     |    S ru   r,   r   s    r"   r   zsplit_module.<locals>.<lambda>  s    {1~ r$   )call_moduler=   r>   zOperator target r?   _)r@   rG   r:   r   r;   r   zMissing exit noder9   c              3   B   K   | ]  }j                   |        y wru   )r   )rx   r   
orig_nodesro   s     r"   rz   zsplit_module.<locals>.<genexpr>  s&      
8<I!!*T"23
s   c              3   (   K   | ]	  }|     y wru   r,   )rx   r   r6   s     r"   rz   zsplit_module.<locals>.<genexpr>&  s     B,t$Bs   c                 "    | j                      S ru   )r   )r   r6   s    r"   r   zsplit_module.<locals>.<lambda>5  s    |AFF?S r$   zpost split_module)Or[   r\   r   r   r   r+   r   r   graph_moduler   sympyr   amp_enter_autocast_exit_autocast_C_set_grad_enabledr	   setr   nodesrH   r^   rv   SymIntr5   exprSymbolr@   rG   rA   r:   boolalladdremovevaluesitemsr_   _LOGGERisEnabledForloggingDEBUGr   map_argr   listkeysr   rm   popr   RuntimeErrorr   rE   tuplerF   rI   r   r9   rK   rV   rJ   rL   rM   joinr    r   dictreversedr   r   rq   r   proxyProxy	enumeratenextiter)Er-   r.   r/   r0   r1   r2   rQ   r   rp   GLOBAL_STATE_NODESgrad_regionsautocast_regionsautocast_exitsactive_gradactive_autocastsvals0akr|   assert_monotonically_increasinghighest_partitionpidoriginal_partition_orderroot_partitionsrn   sorted_partitionsroot_partition	dependentregions_mappingregionsrnew_nodeinpr9   gathered_argsgathered_kwargsrG   target_atomstarget_attrrO   qualnamer   	exit_nodeorig_mod_envr7   construct_order_partitionsalready_constructed_attr_nodesr5   original_orderoutput_valsnum_output_valsrY   orig_mod_attr_nodesbased_mod_attrs
output_valnum_outputsoutput_val_proxyioutput_nameretr6   rP   r   r   ro   ri   rk   rj   sE   ` `  `                                          `            @@@@@@@@r"   r   r   +   sn   P II4	
,,39o, S%(("7"7"C"CCD,8 ')J"$J/1N-C-C"*4.-C^
, 			!!		  "" 1<L 5@M13NKu +: IIMM/22S?sELL)SXX]]*rELL9n$,0N388==)77;;*4077o%$++9K*K{{ehh888499~***!$))A,555",/1E0F,G[)		 9 99J		JJJJ $$T*),nT.B-C)D &'+t$		 8 88499~*** 1.22>$3GH ''		!5/3tyy|,"%)).*>?! 	:AQ##N4$89	:U+:Z >n&;&;&=>>T@TT>1A1G1G1IJA6!9JJ-9-?-?-ABTQAvayLBLBGMM*,.>?(,7&*+;&<&R\@R#   $
499 771177hHHNN""		!K * &C$+ F./seEF+ !$ ;;00HHNN""		V HHNN""X5<  $JOO$56!#O%/%5%5%7 3!	9))*"">23
 $&
(,,.  0#N3>> 	2Iy!..22>Bi(55&&y1	2  Z0=>> -l; 7,224 	7MD'w<!###<@Js71:'33D9QR[ 
7&s1v.	$??66ww;;8dii88"ii 7  !%		 0.6	%%d+
7	77" , A~.	## 	AC#//55$S/.. 6 K  *#3388:K5@I!!*S/2	AA  *34)"4#5#56I $//K!HHNN22499>VWM#hhnn445O ww99#{{005( =D";5,/?}K-XYY")+t"<K=
 ,/,7	!!&)+ #,"7"7!8&BH-1[[L*mU333ot444 7499TD 2277"&)) 3 H !IINN,HM*2I!!$'U*3Z -- 6_- 	6D%d+Gw<!###Sb\ 6&s1v.	*40	 ,A.AA,$??66 ||$++#//57'nn 7  !* 3 3 56	66$ %'L$&L+088>>+?+?+ANCENGGMM 	D+:lN,(L.	 GGMM 	+D&*L#	+ "5:R  &)U" ()ww}}Qt=8PdQNQ4 7E~.	  
@I@Q@Q
 

 k*aOO"";q>2q OO"";/ .7-=-=/&)NAZS!/ / ' 9990?,1-o /22489 , 999/>,0,n /22489 160E0E0Q0Qy1
y,,-
 $//!!B1A1ABB


 )++,?$xx~~33J?"+I,=,="> E;,<Q,?,D,D[)EA:DLd9#4#4567o7Er  77h!!&&tyy|5ST ((


+
+NN
KCIId	
 J} KBj R&/s$   &y)y/:y5y5	y:(y:)NFF) rB   typingr   r   r   r   r   r   collectionsr	   r   r   torch.fx._compatibilityr
   torch.fx.graph_moduler   torch.fx.noder   torch.fx._utilsr   __all__	getLoggerr(   r[   r   r   nnModuleintr+   r   r   r,   r$   r"   <module>r      s     ; ; #   1 -  2 
'!!!(+ +gd+
 
 ,
2 d+
 .2*/.3SSHHOOS dVS[)S 4S>*	S
 "$S &d^S ,Sr$   