
    sg2              	       t   U d dl Z d dlmZmZmZmZmZ d dlZd dlm	Z
 d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ dd	lmZmZmZmZmZmZmZ dd
lmZmZm Z m!Z!m"Z"m#Z# ddl$m%Z%m&Z& dgZ'ee(   e)d<   dejT                  defdZ+dedefdZ,dedede!fdZ-de(dede"fdZ.de(dedede"fdZ/de(dejT                  de"fdZ0de(defdZ1d Z2d Z3de(dedee   dee   fdZ4d ede fd!Z5de(d"edee"   fd#Z6dedefd$Z7dejT                  dee   fd%Z8de(d&ed'edee   fd(Z9d ee(ef   defd)Z:d*ed+ed,ed-efd.Z;y)/    N)AnyCallablecastDictList)_get_device_module)ShardMetadata)ShardedTensor)DTensor)%compute_local_shape_and_global_offset   )BytesStorageMetadataChunkStorageMetadataMetadataIndexSTATE_DICT_TYPESTORAGE_TYPESTensorPropertiesTensorStorageMetadata)LoadItemTypeReadItemSavePlanTensorWriteData	WriteItemWriteItemType)"_check_shard_metadata_pair_overlap+_shards_get_overlap_region_wrt_saved_tensor create_read_items_for_chunk_list__all__tensorreturnc           	          t        t        j                  dgt        | j	                               z        | j	                               S )Nr   offsetssizes)r   torchSizelensize)r   s    _/var/www/html/venv/lib/python3.12/site-packages/torch/distributed/checkpoint/planner_helpers.py_create_chunk_from_tensorr*   '   s5    

A3V[[]!334FKKM     shard_mdc                     t        t        j                  | j                        t        j                  | j                              S Nr"   )r   r%   r&   shard_offsetsshard_sizes)r,   s    r)   _chunk_for_shardr1   -   s3    

8112jj--. r+   sharded_tensorc                    | j                         j                  }t        |j                  |j                  |j
                  |j                  |j                        }t        t        |      || j                         j                        S )N)dtypelayoutrequires_gradmemory_format
pin_memorychunk
propertiesr(   )metadatatensor_propertiesr   r4   r5   r6   r7   r8   r   r1   r(   )r2   r,   shard_propertiesr;   s       r)   _sharded_tensor_metadatar?   4   s|     &..0BB!$$&&&44&44#..J x($$&++ r+   fqnc                    t        |j                  |j                  |j                        \  }}t	        j
                  |      t	        j
                  |      }}t        t        | |      t        j                  t        t        ||      t        j                  |j                               |j                                     S )Nr"   r9   indextypetensor_data)r   shapedevice_mesh
placementsr%   r&   r   r   r   SHARDr   r   r   create_from_tensorto_localr(   )r@   r   r$   r#   s       r)   _create_write_items_for_dtensorrL   H   s    :f((&*;*;NE7 ZZ&

7(;7EC)  #& (::6??;LM
 r+   c                     t        j                  |j                        }t        t	        | |      t
        j                  t        ||            S )NrB   )r%   r&   r/   r   r   r   rI   r?   )r@   r2   r,   r#   s       r)   _create_write_item_for_shardrN   \   sB     jj//0GC)  ,^XF r+   c                 :   t        j                  dgt        |j                               z        }t	        t        | |      t        j                  t        t        ||j                               t        j                  |      |j                                     S )Nr   r"   r9   rB   )r%   r&   r'   r(   r   r   r   TENSORr   r   r   rJ   )r@   r   r#   s      r)   _create_write_item_for_tensorrQ   g   sq    jj!s6;;=112GC)!!#&wfkkmL'::6B
 r+   bytesc                 J    t        t        |       t        j                        S )N)rC   rD   )r   r   r   BYTE_IO)r@   rR   s     r)   _create_write_item_for_bytesiorU   t   s     C "" r+   c           
          t        t        j                  | t        j                  |f      |t        j                  |f      t        j                  |f            S N)rD   
dest_indexdest_offsetsstorage_indexstorage_offsetslengths)r   r   rT   r%   r&   rX   dest_offsetrZ   storage_offsetlengths        r)   _create_read_item_for_byteiora   {   sK     !!ZZ/#

N#45

F9% r+   c           
          t        t        j                  | t        j                  |      |t        j                  |      t        j                  |            S rW   )r   r   rP   r%   r&   rX   rY   rZ   r[   r\   s        r)   _create_read_item_for_tensorrd      sD       ZZ-#

?3

7# r+   checkpoint_mdlocal_chunksc                    g }t        |      D ]  \  }}t        |j                        D ]  \  }}t        ||      sg }g }	g }
t        ||      D ]:  \  }}}}|j	                  |       |	j	                  |       |
j	                  |       < |j	                  t        t        | |j                  |      |	t        | |j                  |      ||
               |S )aW  
    Create a list of ``ReadItem`` based on the checkpoint and local chunks.

    This applies the resharding algorithm and computes the reads needed
    to satisfy ``local_chunks`` with a checkpoint described by ``checkpoint_md``.

    Args:
        fqn (str) : The state_dict FQN to pass to ``ReadItem``.
        checkpoint_md (TensorStorageMetadata): metadata for a given tensor
            from a checkpoint.
        local_chunks (List[ChunkStorageMetadata]): Local chunks that needs to be
            loaded.

    Returns:
        A list of ``ReadItem`` that will satisfy all input chunks.
    )saved_shardcurrent_shardrc   )	enumeratechunksr   r   appendrd   r   r#   )r@   re   rf   
read_itemsidxshardstorage_idx
storage_mdr[   rY   r\   dimoffset_for_saved_tensoroffset_for_current_tensorr`   s                  r)   r   r      s    * J- 
U'01E1E'F 	#K5eZH OLG =&e
' ')  &&'>?##$=>v&
' ,,S%--E!-"/Z5G5G"U$3#'	: r+   
state_dictc           	         g }| j                         D ]  \  }}t        |t              r|j                  t	        ||             2t        |t
              r<|j                         j                  D ]  }|j                  t        |||               ~t        |t        j                        r|j                  t        ||             |j                  t        ||              t        |      S )N)items
isinstancer   rl   rL   r
   r<   shards_metadatarN   r%   TensorrQ   rU   r   )ru   requestsr@   objr,   s        r)   "_create_default_metadata_only_planr}      s    H$$& 	FSc7#OO;CEF]+LLN:: R <S#x PQRU\\*OO9#sCDOO:3DE	F Hr+   objectc                 6   t        |d      r|j                  | |      S t        |t              r3|j	                         D cg c]  }t        | ||j                         c}S t        |t        j                        rt        | |      gS t        | |      gS c c}w )N__create_write_items__)hasattrr   rx   r
   local_shardsrN   r<   r%   rz   rQ   rU   )r@   r~   ro   s      r)   _create_write_itemsr      s    v/0,,S&99	FM	*  ,,.
 )fennE
 	
 
FELL	)-c6:;;.sF;<<
s   Bc                     t        | j                  | j                  | j                        \  }}t	        j
                  |      t	        j
                  |      }}t        ||      S r.   )r   rF   rG   rH   r%   r&   r   )r   r$   r#   s      r)   _create_chunk_from_dtensorr      sW    :f((&*;*;NE7 ZZ&

7(;7E r+   c                 J   t        | d      r| j                         }|S t        | t              r2| j	                         D cg c]  }t        |j                         }}|S t        | t        j                        rt        |       g}|S t        dt        |              c c}w )N__create_chunk_list__zMUnsupported Type, expecting one of [Tensor, DTensor, ShardedTensor] ,but got )r   r   rx   r
   r   r1   r<   r%   rz   r*   
ValueErrorrD   )r   rf   ro   s      r)   _create_chunk_listr      s    v./335  
FM	*:@:M:M:O
16U^^,
 
  
FELL	)1&9:  V~'
 	

s   B mdr|   c                     t        |t              s	 t        |      }t        | ||      S t        t        |       dt        |       dd      gS # t        $ r$}t        d|  ddt	        |       z         |d }~ww xY w)Nz Invalid checkpoint metadata for z, z(expected BytesStorageMetadata but found r   r]   )rx   r   r   r   rD   r   ra   r   )r@   r   r|   rf   exs        r)   _create_read_itemsr     s    b./	-c2L 0RFF )(-+C0 
 	
  	23%r:<T"XJGH 	s   A 	A:A55A:c                 p    dt         fd}dt        fd}dt        j                  fd}t	        | |||       y)zP
    Initializes meta tensor if the meta tensor is DTensor or torch.Tensor.
    valuec                    t        | dd       }|t        j                  d      k(  rt        j                  j                         j                  }t        t        j                  t        |      j                               }t        j                  | j                         |      }t        j                  || j                  | j                  | j!                         | j#                               }|S | S )Ndevicemetar   )rG   rH   rF   stride)getattrr%   r   distdistributed_c10d_get_pg_default_devicerD   r   r   current_device
empty_likerK   r   
from_localrG   rH   r(   r   )r   r   device_typenew_local_tensordtensors        r)   dtensor_funcz&_init_state_dict.<locals>.dtensor_func$  s    $/U\\&))//FFHMMK0=LLNF  %//0@P (( !-- ++jjl||~G NLr+   c                     t        | dd       }|t        j                  d      k(  rt        dt	        |        d      | S )Nr   r   zFound unsupported type z for meta device loading.)r   r%   r   RuntimeErrorrD   )r   r   s     r)   sharded_tensor_funcz-_init_state_dict.<locals>.sharded_tensor_func9  sF    $/U\\&)))$u+6OP  Lr+   c                 4   t        | dd       }|t        j                  d      k(  rrt        j                  j                         j                  }t        t        j                  t        |      j                               }t        j                  | |      }|S | S )Nr   r   r   )r   r%   r   r   r   r   rD   r   r   r   r   )r   r   r   r   s       r)   tensor_funcz%_init_state_dict.<locals>.tensor_funcB  s{    $/U\\&))//FFHMMK0=LLNF %%eF;FMLr+   N)r   r   r%   rz   _iterate_state_dict)ru   r   r   r   s       r)   _init_state_dictr     s@    
G *3 
5<< 
 	r+   iter_objectr   r   r   c           	      *   t        | t              r ||       S t        | t              r ||       S t        | t        j                        r ||       S t        | t
        t        t        t        t        j                  f      s| | S t        | t              r+| j                         D ]  \  }}t        ||||      | |<    | S t        | t        t        f      r8| D cg c]  }t        ||||       }}t        | t              rt        |      }|S yc c}w )a$  
    Iterate through the state dict, applying the given functions to each tensor type
    and update the state dict in place.

    Args:
        iter_object (Any): the target state_dict.
        sharded_tensor_func (Callable): the function to apply to ShardedTensor
        dtensor_func (Callable): the function to apply to DTensor
        tensor_func (Callable): the function to apply to Tensor

    # TODO: let state_dict_util._iterate_state_dict() to support in place option
    so we don't need to have two versions of _iterate_state_dict.
    N)rx   r   r
   r%   rz   intfloatstrrR   ioBytesIOdictrw   r   listtuple)r   r   r   r   keyr   vrets           r)   r   r   V  s   ( +w'K((	K	/";//	K	.;'';eS% DE	K	&%++- 	JC2|%8+ K	 	K$	/ !
  <1DkR
 
 k5)*C
 
0
s   D)<r   typingr   r   r   r   r   r%   torch.distributeddistributedr   torch._utilsr   !torch.distributed._shard.metadatar	   'torch.distributed._shard.sharded_tensorr
   torch.distributed.tensorr   torch.distributed.tensor._utilsr   r<   r   r   r   r   r   r   r   plannerr   r   r   r   r   r   
reshardingr   r   r   r   __annotations__rz   r*   r1   r?   rL   rN   rQ   rU   ra   rd   r   r}   r   r   r   r   r   r    r+   r)   <module>r      s   	 2 2    + ; A , Q    99c 9ell 7K } 1E !-:( g ) (	+7D
s 
ELL 
Y 
 C 

4	4(4 +,4 
(^	4n? x =S =# =$y/ =w 3G u|| 5I0J &
C 
] 
 
h 
.4c3h 4C 4n,,, ", 	,r+   