
    sg                         d dl Z d dlZd dlmZ d dlmZ d dlmZmZm	Z	 d dl
Z
d dl
mZ d dlmZ d dlmZmZmZ dd	gZ G d
 de      Z G d de      Z G d d	e      Zy)    N)contextmanager)Path)	GeneratorOptionalUnion)AbstractFileSystem)	url_to_fs)FileSystemBaseFileSystemReaderFileSystemWriterFsspecWriterFsspecReaderc            
          e Zd ZddZedeeej                  f   dede	e
j                  ddf   fd       Zdeeej                  f   dedeeej                  f   fdZdeeej                  f   deeej                  f   fd	Zdeeej                  f   d
eeej                  f   ddfdZdeej                  gddfdZedeeej                  f   defd       Zdeeej                  f   defdZdeeej                  f   ddfdZy)
FileSystemreturnNc                     d | _         y N)fs)selfs    b/var/www/html/venv/lib/python3.12/site-packages/torch/distributed/checkpoint/_fsspec_filesystem.py__init__zFileSystem.__init__   s	    04    pathmodec              #      K   | j                   J | j                   j                  5  t        j                  t	        |      |      5 }| d d d        d d d        y # 1 sw Y   xY w# 1 sw Y   y xY wwr   )r   transactionfsspecopenstr)r   r   r   streams       r   create_streamzFileSystem.create_stream   sl      ww"""WW   	SY- 	 	 	 	s4   %A5 A)AA)	A5A&	"A))A2.A5suffixc                 B    t         j                  j                  ||      S r   )osr   join)r   r   r"   s      r   concat_pathzFileSystem.concat_path(   s     ww||D&))r   c                 ,    t        |      \  | _        }|S r   )r	   r   )r   r   _s      r   	init_pathzFileSystem.init_path-   s    t_
r   new_pathc                 <    | j                   j                  ||       y r   )r   rename)r   r   r*   s      r   r,   zFileSystem.rename1   s     	tX&r   c                 >    | j                   j                  |d       y )NT)exist_ok)r   makedirsr   r   s     r   mkdirzFileSystem.mkdir6   s    -r   checkpoint_idc                 \    t        |t              ry	 t        |       y# t        $ r Y yw xY w)NFT)
isinstancer   r	   
ValueErrorclsr2   s     r   validate_checkpoint_idz!FileSystem.validate_checkpoint_id9   s5    mT*	m$   		s    	++c                 8    | j                   j                  |      S r   )r   existsr0   s     r   r:   zFileSystem.existsE   s    ww~~d##r   c                 :    | j                   j                  |       y r   )r   rmr0   s     r   rm_filezFileSystem.rm_fileH   s    

4r   )r   N)__name__
__module____qualname__r   r   r   r   r$   PathLiker   ioIOBaser!   r&   r)   r,   r1   classmethodboolr8   r:   r=    r   r   r   r      s   5 #r{{*+36	299dD(	) *#r{{*+*58*	sBKK	 *
eC$45 %R[[@P:Q '#r{{*+'7<S"++=M7N'	'
.3, . . 	5bkk9I3J 	t 	 	$5bkk!12 $t $E#r{{"23  r   r   c                        e Zd ZdZ	 	 	 	 	 ddeeej                  f   dedede	de	dedd	f fd
Z
edeeej                  f   defd       Z xZS )r   a`  
    Basic implementation of StorageWriter using FFspec.

    This implementation makes the following assumptions and simplifications:

    * The checkpoint path is an empty or non-existing directory.
    * File creation is atomic

    The checkpoint consist of one file per write request plus
    a `.metadata` file with the serialized metadata.

    r   single_file_per_rank
sync_filesthread_countper_thread_copy_ahead	overwriter   Nc                     t         |   ||||||       t               | _        | j                  j	                  |      | _        y)a  
        Initialize the writer pointing to `path`.

        Args:
            path: directory where the checkpoint will be written to.
            single_file_per_rank: Produce one file per rank instead of one file per tensor/blob. Default to True.
            sync_files : force files to be synced to permanent storage. Default to True.
            thread_count: Number of IO threads to use to write. Default to 1.
            per_thread_copy_ahead: How many bytes to copy from the GPU ahead of saving then. Default 10Mb.
            overwrite: Whether to allow overwriting existing checkpoints. Defaults to True.

        N. B. If sync_files is disabled, there's no guarantee that the checkpoint will be consistent in the case of a failure.
        )rL   Nsuperr   r   r   r)   r   )r   r   rH   rI   rJ   rK   rL   	__class__s          r   r   zFsspecWriter.__init__[   sJ    , 	 ! 	 	
 ,GG%%d+	r   r2   c                 ,    t         j                  |      S r   r   r8   r6   s     r   r8   z#FsspecWriter.validate_checkpoint_id|       00??r   )TT   i T)r>   r?   r@   __doc__r   r   r$   rA   rE   intr   rD   r8   __classcell__rP   s   @r   r   r   M   s      &*%/,C$%, #, 	,
 ,  #, , 
,B @5bkk9I3J @t @ @r   c                   ~     e Zd Zdeeej                  f   ddf fdZedeeej                  f   de	fd       Z
 xZS )r   r   r   Nc                     t         |   |       t               | _        | j                  j	                  |      | _        y r   rN   )r   r   rP   s     r   r   zFsspecReader.__init__   s/    ,GG%%d+	r   r2   c                 ,    t         j                  |      S r   rR   r6   s     r   r8   z#FsspecReader.validate_checkpoint_id   rS   r   )r>   r?   r@   r   r   r$   rA   r   rD   rE   r8   rW   rX   s   @r   r   r      sX    ,U3#34 , ,
 @5bkk9I3J @t @ @r   )rB   r$   
contextlibr   pathlibr   typingr   r   r   r   r   fsspec.corer	   'torch.distributed.checkpoint.filesystemr
   r   r   __all__r   r   r   rF   r   r   <module>rb      sg    
 	 %  - -  % !  . .d1@# 1@h@# @r   