
    sgA                        d Z ddlZddlZddlZddlZddlZddlmZ ddlm	Z	 ddl
mZmZmZ ddlZddlmZ dd	lmZ  ej&                  e      Zd
Z G d de      Z G d d      ZdefdZdee   fdZ G d de	      Z G d d      Zy)a  
Implementation of a custom transfer agent for the transfer type "multipart" for git-lfs.

Inspired by: github.com/cbartz/git-lfs-swift-transfer-agent/blob/master/git_lfs_swift_transfer.py

Spec is: github.com/git-lfs/git-lfs/blob/master/docs/custom-transfers.md


To launch debugger while developing:

``` [lfs "customtransfer.multipart"]
path = /path/to/transformers/.env/bin/python args = -m debugpy --listen 5678 --wait-for-client
/path/to/transformers/src/transformers/commands/transformers_cli.py lfs-multipart-upload ```    N)ArgumentParser)AbstractContextManager)DictListOptional   )logging   )BaseTransformersCLICommandzlfs-multipart-uploadc                   &    e Zd ZdZedefd       Zy)LfsCommandsu  
    Implementation of a custom transfer agent for the transfer type "multipart" for git-lfs. This lets users upload
    large files >5GB 🔥. Spec for LFS custom transfer agent is:
    https://github.com/git-lfs/git-lfs/blob/master/docs/custom-transfers.md

    This introduces two commands to the CLI:

    1. $ transformers-cli lfs-enable-largefiles

    This should be executed once for each model repo that contains a model file >5GB. It's documented in the error
    message you get if you just try to git push a 5GB file without having enabled it before.

    2. $ transformers-cli lfs-multipart-upload

    This command is called by lfs directly and is not meant to be called by the user.
    parserc                     | j                  dd      }|j                  dt        d       |j                  d        | j                  t        d	      }|j                  d
        y )Nzlfs-enable-largefileszeDeprecated: use `huggingface-cli` instead. Configure your repository to enable upload of files > 5GB.)helppathz/Local path to repository you want to configure.)typer   c                     t        |       S N)LfsEnableCommandargss    L/var/www/html/venv/lib/python3.12/site-packages/transformers/commands/lfs.py<lambda>z1LfsCommands.register_subcommand.<locals>.<lambda>?       5Ed5K     )funczgDeprecated: use `huggingface-cli` instead. Command will get called by git-lfs, do not call it directly.c                     t        |       S r   )LfsUploadCommandr   s    r   r   z1LfsCommands.register_subcommand.<locals>.<lambda>H   r   r   )
add_parseradd_argumentstrset_defaultsLFS_MULTIPART_UPLOAD_COMMAND)r   enable_parserupload_parsers      r   register_subcommandzLfsCommands.register_subcommand6   sy    ))#w * 
 	""6:k"l""(K"L))(O * 
 	""(K"Lr   N)__name__
__module____qualname____doc__staticmethodr   r&    r   r   r   r   $   s%    " MN M Mr   r   c                       e Zd Zd Zd Zy)r   c                     || _         y r   r   selfr   s     r   __init__zLfsEnableCommand.__init__L   	    	r   c                    t        j                  d       t        j                  j	                  | j
                  j                        }t        j                  j                  |      st        d       t        d       t        j                  dj                         d|       t        j                  dt         j                         d|       t        d       y )	NzcManaging repositories through transformers-cli is deprecated. Please use `huggingface-cli` instead.z)This does not look like a valid git repo.r
   z=git config lfs.customtransfer.multipart.path transformers-cliT)checkcwdz-git config lfs.customtransfer.multipart.args z Local repo set up for largefiles)warningswarnosr   abspathr   isdirprintexit
subprocessrunsplitr#   )r0   
local_paths     r   r>   zLfsEnableCommand.runO   s    q	
 WW__TYY^^4
ww}}Z(=>GKQQS[_eo	
 	;<X;YZ``b	

 	01r   Nr'   r(   r)   r1   r>   r,   r   r   r   r   K   s    2r   r   msgc                     t        j                  |       dz   } t        j                  j	                  |        t        j                  j                          y)z-Write out the message in Line delimited JSON.
N)jsondumpssysstdoutwriteflushrB   s    r   	write_msgrL   b   s6    
**S/D
 CJJSJJr   returnc                  H   t        j                  t        j                  j	                         j                               } d| j                  d      | j                  d      fv ry| j                  d      dvr*t        j                  d       t        j                  d       | S )z$Read Line delimited JSON from stdin.	terminater   eventN)downloaduploadzReceived unexpected messager
   )
rE   loadsrG   stdinreadlinestripgetloggercriticalr<   rK   s    r   read_msgrZ   i   sv    
**SYY'')//1
2Cswwv(899
www5556Jr   c                   D    e Zd ZdZdededefdZd Zd ZddZ	d	 Z
d
 Zy)	FileSlicezq
    File-like object that only reads a slice of a file

    Inspired by stackoverflow.com/a/29838711/593036
    filepath	seek_from
read_limitc                 <    || _         || _        || _        d| _        y )Nr   )r]   r^   r_   n_seen)r0   r]   r^   r_   s       r   r1   zFileSlice.__init__   s     "$r   c                     t        | j                  d      | _        | j                  j                  | j                         | S )Nrb)openr]   fseekr^   r0   s    r   	__enter__zFileSlice.__enter__   s-    dmmT*DNN#r   c                     t        j                  | j                  j                               j                  }t        | j                  || j                  z
        S r   )r8   fstatre   filenost_sizeminr_   r^   )r0   total_lengths     r   __len__zFileSlice.__len__   s:    xx0884??L4>>$ABBr   c                     | j                   | j                  k\  ry| j                  | j                   z
  }| j                  j                  |dk  r|nt	        ||            }| xj                   t        |      z  c_         |S )Nr   r   )ra   r_   re   readrm   len)r0   nremaining_amountdatas       r   rq   zFileSlice.read   sb    ;;$//)??T[[8vv{{q1u+#aAQ:RSs4y r   c              #   4   K   | j                  d       y w)Ni  @ )rs   )rq   rg   s    r   __iter__zFileSlice.__iter__   s     ii/i**s   c                 8    | j                   j                          y r   )re   closer/   s     r   __exit__zFileSlice.__exit__   s    r   N))r'   r(   r)   r*   r!   intr1   rh   ro   rq   rw   rz   r,   r   r   r\   r\   x   s;      # 
C+r   r\   c                       e Zd Zd Zd Zy)r   c                     || _         y r   r   r/   s     r   r1   zLfsUploadCommand.__init__   r2   r   c           	         t        j                  t        j                  j	                         j                               }|j                  d      dk(  r|j                  d      dk(  s%t        ddddi       t        j                  d	       t        i        	 t               }|t        j                  d
       |d   }|d   }|d   d   }|d   d   }t        |j                  d            }t        |j                               }g }	t        |      D ]  \  }
}t        ||
|z  |      5 }t!        j"                  ||      }|j%                          |	j'                  |j(                  j                  d      |
d	z   d       t        d||
d	z   |z  |d       d d d         t!        j*                  |||	d      }|j%                          t        d|d       D# 1 sw Y   xY w)NrP   init	operationrR   error    zWrong lfs init operation)codemessager
   r   oidr   actionhrefheader
chunk_size)r^   r_   )ru   etag)r   
partNumberprogress)rP   r   
bytesSoFarbytesSinceLast)r   parts)rE   complete)rP   r   )rE   rS   rG   rT   rU   rV   rW   rL   r<   rZ   r|   poplistvalues	enumerater\   requestsputraise_for_statusappendheaderspost)r0   init_msgrB   r   r]   completion_urlr   r   presigned_urlsr   ipresigned_urlru   rs                 r   r>   zLfsUploadCommand.run   s    ::cii00288:;W%/HLL4MQY4Yw8R STUHHQK 	" *C{
 e*C6{H ]62N]8,FVZZ56J(,V]]_(=NE$-n$=  =x1z>jY ]a ]>A&&(LL$%IIMM&$9*+a% %/#&+,q5J*>.8	 , "A  
378a $ s   +A.GG&	NrA   r,   r   r   r   r      s    A9r   r   )r*   rE   r8   r=   rG   r6   argparser   
contextlibr   typingr   r   r   r   utilsr	    r   
get_loggerr'   rX   r#   r   r   rL   rZ   r\   r   r,   r   r   <module>r      s   `  	  
  # - ' '   ( 
		H	%  6 $M, $MN2 2.4 (4. "& "JE9 E9r   