U
    ”Ê¹j>  ã                   @   sÖ   d Z ddlZddlZddlZddlZddlmZ ddlmZm	Z	m
Z
 ddlmZ ddlmZ ddlmZmZmZ dd	lmZ e e¡ZG d
d„ deƒZG dd„ dƒZedœdd„Ze
e dœdd„ZG dd„ dƒZdS )a  
Implementation of a custom transfer agent for the transfer type "multipart" for
git-lfs.

Inspired by:
github.com/cbartz/git-lfs-swift-transfer-agent/blob/master/git_lfs_swift_transfer.py

Spec is: github.com/git-lfs/git-lfs/blob/master/docs/custom-transfers.md


To launch debugger while developing:

``` [lfs "customtransfer.multipart"]
path = /path/to/huggingface_hub/.env/bin/python args = -m debugpy --listen 5678
--wait-for-client
/path/to/huggingface_hub/src/huggingface_hub/commands/huggingface_cli.py
lfs-multipart-upload ```é    N)Ú_SubParsersAction)ÚDictÚListÚOptional)ÚBaseHuggingfaceCLICommand)ÚLFS_MULTIPART_UPLOAD_COMMANDé   )Úget_sessionÚhf_raise_for_statusÚlogging)ÚSliceFileObjc                   @   s"   e Zd ZdZeedœdd„ƒZdS )ÚLfsCommandsu®  
    Implementation of a custom transfer agent for the transfer type "multipart"
    for git-lfs. This lets users upload large files >5GB ðŸ”¥. Spec for LFS custom
    transfer agent is:
    https://github.com/git-lfs/git-lfs/blob/master/docs/custom-transfers.md

    This introduces two commands to the CLI:

    1. $ hf lfs-enable-largefiles

    This should be executed once for each model repo that contains a model file
    >5GB. It's documented in the error message you get if you just try to git
    push a 5GB file without having enabled it before.

    2. $ hf lfs-multipart-upload

    This command is called by lfs directly and is not meant to be called by the
    user.
    )Úparserc                 C   sP   | j ddd�}|jdtdd� |jdd„ d	� | j tdd�}|jd
d„ d	� d S )Nzlfs-enable-largefilesF)Úadd_helpÚpathz/Local path to repository you want to configure.)ÚtypeÚhelpc                 S   s   t | ƒS ©N)ÚLfsEnableCommand©Úargs© r   úV/var/www/html/bwave/python/venv/lib/python3.8/site-packages/huggingface_hub/cli/lfs.pyÚ<lambda>=   ó    z1LfsCommands.register_subcommand.<locals>.<lambda>)Úfuncc                 S   s   t | ƒS r   )ÚLfsUploadCommandr   r   r   r   r   A   r   )Ú
add_parserÚadd_argumentÚstrÚset_defaultsr   )r   Zenable_parserZupload_parserr   r   r   Úregister_subcommand9   s
    zLfsCommands.register_subcommandN)Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ústaticmethodr   r!   r   r   r   r   r   $   s   r   c                   @   s   e Zd Zdd„ Zdd„ ZdS )r   c                 C   s
   || _ d S r   r   ©Úselfr   r   r   r   Ú__init__E   s    zLfsEnableCommand.__init__c                 C   sf   t j | jj¡}t j |¡s,tdƒ tdƒ tjd 	¡ d|d� tjdt
› � 	¡ d|d� tdƒ d S )Nz)This does not look like a valid git repo.é   z/git config lfs.customtransfer.multipart.path hfT)ÚcheckÚcwdz-git config lfs.customtransfer.multipart.args z Local repo set up for largefiles)Úosr   Úabspathr   ÚisdirÚprintÚexitÚ
subprocessÚrunÚsplitr   )r(   Z
local_pathr   r   r   r3   H   s    ýýzLfsEnableCommand.runN©r"   r#   r$   r)   r3   r   r   r   r   r   D   s   r   ©Úmsgc                 C   s(   t  | ¡d }tj |¡ tj ¡  dS )z-Write out the message in Line delimited JSON.Ú
N)ÚjsonÚdumpsÚsysÚstdoutÚwriteÚflush)r7   Zmsg_strr   r   r   Ú	write_msgZ   s    r?   ©Úreturnc                  C   sV   t  tj ¡  ¡ ¡} d|  d¡|  d¡fkr0dS |  d¡dkrRt d¡ t 	d¡ | S )z$Read Line delimited JSON from stdin.Ú	terminater   ÚeventN)ÚdownloadÚuploadzReceived unexpected messager*   )
r9   Úloadsr;   ÚstdinÚreadlineÚstripÚgetÚloggerÚcriticalr1   r6   r   r   r   Úread_msga   s    

rM   c                   @   s(   e Zd Zddœdd„Zddœdd„ZdS )r   Nr@   c                 C   s
   || _ d S r   r   r'   r   r   r   r)   q   s    zLfsUploadCommand.__init__c                 C   sŠ  t  tj ¡  ¡ ¡}| d¡dkr0| d¡dksLtddddœiƒ t d	¡ ti ƒ t	ƒ }|d krlt d
¡ |d }|d }|d d }|d d }t
| d¡ƒ}t| ¡ ƒ}td|d	d
dœƒ g }	t|dƒ�„}
t|ƒD ]t\}}t|
|| |d��T}tƒ j||d�}t|ƒ |	 |j d¡|d	 dœ¡ td||d	 | |dœƒ W 5 Q R X qØW 5 Q R X tƒ j|||	dœd�}t|ƒ td|dœƒ qTd S )NrC   ÚinitZ	operationrE   Úerroré    zWrong lfs init operation)ÚcodeÚmessager*   r   Úoidr   ÚactionÚhrefÚheaderÚ
chunk_sizeÚprogress)rC   rS   Z
bytesSoFarZbytesSinceLastÚrb)Z	seek_fromZ
read_limit)ÚdataÚetag)r[   Z
partNumber)rS   Úparts)r9   Zcomplete)rC   rS   )r9   rF   r;   rG   rH   rI   rJ   r?   r1   rM   ÚintÚpopÚlistÚvaluesÚopenÚ	enumerater   r	   Úputr
   ÚappendÚheadersÚpost)r(   Zinit_msgr7   rS   ÚfilepathZcompletion_urlrV   rW   Zpresigned_urlsr\   ÚfileÚiZpresigned_urlrZ   Úrr   r   r   r3   t   sh    

üÿ	ý
þÿ
üÿ
þþzLfsUploadCommand.runr5   r   r   r   r   r   p   s   r   )r%   r9   r-   r2   r;   Úargparser   Útypingr   r   r   Zhuggingface_hub.commandsr   Zhuggingface_hub.lfsr   Úutilsr	   r
   r   Z
utils._lfsr   Z
get_loggerr"   rK   r   r   r?   rM   r   r   r   r   r   Ú<module>   s    
 