U ʹjҼ@sddlZddlZddlZddlmZddlmZmZddlmZddl m Z ddl m Z ddl mZmZmZmZmZmZmZmZddlmZmZddlZdd lmZmZmZdd lmZdd l m!Z!d d l"m#Z#d dl$m%Z%m&Z&d dl'm(Z(m)Z)m*Z*d dl+m,Z,m-Z-d dl.m/Z/m0Z0m1Z1d dl2m3Z3m4Z4m5Z5e6dej7Z8eGdddZ9Gdddej:Z;GdddejGdddejZ)d?d@Z*dAdBZ+Z,S)M HfFileSystema% Access a remote Hugging Face Hub repository as if were a local file system. > [!WARNING] > [`HfFileSystem`] provides fsspec compatibility, which is useful for libraries that require it (e.g., reading > Hugging Face datasets directly with `pandas`). However, it introduces additional overhead due to this compatibility > layer. For better performance and reliability, it's recommended to use `HfApi` methods when possible. Args: token (`str` or `bool`, *optional*): A valid user access token (string). Defaults to the locally saved token, which is the recommended method for authentication (see https://huggingface.co/docs/huggingface_hub/quick-start#authentication). To disable authentication, pass `False`. endpoint (`str`, *optional*): Endpoint of the Hub. Defaults to . Usage: ```python >>> from huggingface_hub import HfFileSystem >>> fs = HfFileSystem() >>> # List files >>> fs.glob("my-username/my-model/*.bin") ['my-username/my-model/pytorch_model.bin'] >>> fs.ls("datasets/my-username/my-dataset", detail=False) ['datasets/my-username/my-dataset/.gitattributes', 'datasets/my-username/my-dataset/README.md', 'datasets/my-username/my-dataset/data.json'] >>> # Read/write files >>> with fs.open("my-username/my-model/pytorch_model.bin") as f: ... data = f.read() >>> with fs.open("my-username/my-model/pytorch_model.bin", "wb") as f: ... f.write(data) ``` r0ZhfN)endpointtoken block_sizecsDtj|||ptj|_||_t||d|_||_i|_ i|_ dS)N)rDrE) super__init__rZENDPOINTrDrEr _apirF_repo_and_revision_exists_cachedircache)r8rDrErFargsstorage_options __class__r:r;rHcs zHfFileSystem.__init__)r'r(r)r/c Cs|||f|jkrz|jj|||tjdWnttfk rx}z,d|f|j|||f<d|f|j||df<W5d}~XYnftk r}z(d|f|j|||f<d|j||df<W5d}~XYn"Xd|j|||f<d|j||df<|j|||fS)N)r)r'timeoutF)TN)rJrIZ repo_inforZHF_HUB_ETAG_TIMEOUTrr#r)r8r'r(r)er:r:r;_repo_and_revision_existys" $"z%HfFileSystem._repo_and_revision_exist)pathr)r/cCs|ttttttddd}||}|s6tdnN|dddtjkr~d|krbtd|dd\}}tj|}ntj }| ddkrd|kr>|dd\}}d|krt |}|d k r|d | fkrt d |d}| }n|dd\}}nd }|t||}||||\} } | st|| nd }d|dd d } d|dd d } |dd} d|ddd }| }| }||||\} } | sVt| ttfr| }|}||||\} }| st|| n t|| nV|}d }d|kr2|dd\}}|t||}nd }||||\} }| sVtd|d k rd|ntj}t|||||d S) a Resolve a Hugging Face file system path into its components. Args: path (`str`): Path to resolve. revision (`str`, *optional*): The revision of the repo to resolve. Defaults to the revision specified in the path. Returns: [`HfFileSystemResolvedPath`]: Resolved path information containing `repo_type`, `repo_id`, `revision` and `path_in_repo`. Raises: `ValueError`: If path contains conflicting revision information. `NotImplementedError`: If trying to list repositories. )revision_in_pathr)r/cSs8|dk r0|dk r4||kr4td|d|dn|}|S)NzRevision specified in path ("z ") and in `revision` argument ("z") are not the same.) ValueError)rTr)r:r:r;%_align_revision_in_path_with_revisionszHHfFileSystem.resolve_path.._align_revision_in_path_with_revisionz0Access to repositories lists is not implemented.r2rrr1Nr0)r-)r rAZ_strip_protocolNotImplementedErrorsplitrr3valuesZREPO_TYPES_MAPPINGZREPO_TYPE_MODELcountSPECIAL_REFS_REVISION_REGEXsearchgroupsublstriprrR_raise_file_not_foundjoin isinstancerr#r6r&)r8rSr)rVr'r(rTmatchr*Zrepo_and_revision_existerrZrepo_id_with_namespaceZpath_in_repo_with_namespaceZrepo_id_without_namespaceZpath_in_repo_without_namespace_r:r:r; resolve_pathsl           zHfFileSystem.resolve_path)rSr/cCs|s|j|jnh||}|}|rJ|j|d||}q,|js|j|j|j dfd|j|j|j |j fddS)ac Clear the cache for a given path. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.invalidate_cache). Args: path (`str`, *optional*): Path to clear from cache. If not provided, clear the entire cache. N) rKclearrJrgr<pop_parentr*r'r(r))r8rS resolved_pathr:r:r;invalidate_caches     zHfFileSystem.invalidate_cacherbHfFileSystemFile)rSmoder)rFr/cKsn|dk r |n|j}|dk r"||d<d|kr2td|dkrRt||f||d|St||f||d|SdS)NrFaz/Appending to remote files is not yet supported.r)ror))rFrXHfFileSystemStreamFilern)r8rSror)rFkwargsr:r:r;_openszHfFileSystem._openc KsR|j||d}|jj|j|j|j|j|j|d|dd|j | ddS)Nr)commit_messagecommit_description)r*r(rEr'r)rurvrS) rgrIZ delete_filer*r(rEr'r)r4rlr<)r8rSr)rrrkr:r:r;_rms zHfFileSystem._rmF)rS recursivemaxdepthr)r/c sj||d}j||||d}fdd|D}dd|D} d|d} | |rVdnd 7} | |d k rrd |dnd 7} jj|j|jj| |j|d | |d dj | dd S)a Delete files from a repository. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.rm). > [!WARNING] > Note: When possible, use `HfApi.delete_file()` for better performance. Args: path (`str`): Path to delete. recursive (`bool`, *optional*): If True, delete directory and all its contents. Defaults to False. maxdepth (`int`, *optional*): Maximum number of subdirectories to visit when deleting recursively. revision (`str`, *optional*): The git revision to delete from. rt)ryrzr)cs"g|]}|s|jqSr:)isdirrgr*).0rSr8r:r; Cs z#HfFileSystem.rm..cSsg|]}t|dqS))r*)r)r|r*r:r:r;r~DszDelete  z recursively r0Nz up to depth rurv)r(r'rE operationsr)rurvrw) rgZ expand_pathrI create_commitr(r'rEr)r4rlr<) r8rSryrzr)rrrkpathsZ paths_in_reporrur:r}r;rm&s"   zHfFileSystem.rmT)rSdetailrefreshr)r/c s|j|d}|z|jf||d|}Wnjtk r|jsRtd|j|f||d|}fdd|D}t|dkrtdYnX|r|Sdd|DS)a List the contents of a directory. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.ls). > [!WARNING] > Note: When possible, use `HfApi.list_repo_tree()` for better performance. Args: path (`str`): Path to the directory. detail (`bool`, *optional*): If True, returns a list of dictionaries containing file information. If False, returns a list of file paths. Defaults to True. refresh (`bool`, *optional*): If True, bypass the cache and fetch the latest data. Defaults to False. revision (`str`, *optional*): The git revision to list from. Returns: `List[Union[str, Dict[str, Any]]]`: List of file paths (if detail=False) or list of file information dictionaries (if detail=True). rt)rr)Ncsg|]}|dkr|qSnamer:r|orwr:r;r~ws z#HfFileSystem.ls..rcSsg|] }|dqSrr:rr:r:r;r~zs)rgr<_ls_treerr*rarjlen)r8rSrrr)rrrkoutr:rwr;lsTs  zHfFileSystem.ls)rSryrr) expand_inforzc sj||d}|}t|j|j|jd|jd}g} |jkr|sj|} | | g} |rdt fdd| D} | r| \} |dks|kr| djkr| | dqj| d} | | | fdd| Dqg}|r fd d| D}|r| s$|r|rt j | |}|d sZ||ksZ|t| |krd|d n||dk rt|dd }||8}fd d| D} tjD]$}|d rj|dqjd| j|d |||d nЈjj|j|j|||j|jd}|D]}|d |j }t|trf||jd|j|j |j!|j"d}n|dd|j#|j!d}|d}j$|g ||t|dd |dksƈ|kr(| |q(| S)Nrtr0r*r-rWcs g|]}|ddkr|fqStype directoryr:r| path_infodepthr:r;r~s z)HfFileSystem._ls_tree..rcs$g|]}|ddkrd|fqS)rrrr:rrr:r;r~s cs&g|]}|ddkr|dqS) last_commitNr)rjrr}r:r;r~s r2cs"g|]}|dds|qS)rr2) startswithr) common_pathr:r;r~sT)ryrr)rrz)ryexpandr)r'filersizerblob_idlfsrsecurityrrrrrtree_idr)%rgr<r&r'r(r)r-rKextendrpopleftappendosrS commonprefixendswithrr5rjrr[listrrirrIZlist_repo_treer*rcr"rrrrrr setdefault)r8rSryrr)rrzrk root_pathrZcached_path_infosZdirs_not_in_dircacheZ dirs_to_visitdir_infoZdirs_not_expanded common_prefixZcommon_path_depthZ cached_pathtreer cache_pathZcache_path_info parent_pathr:)rrr8r;r|s            zHfFileSystem._ls_treec/s6|j||dd}tj|f||EdHdS)a Return all files below the given path. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.walk). Args: path (`str`): Root path to list files from. Returns: `Iterator[Tuple[str, List[str], List[str]]]`: An iterator of (path, list of directory names, list of file names) tuples. r)rtN)rgr4r<rGwalk)r8rSrLrrrNr:r;rs zHfFileSystem.walkc s(|j||dd}tj|f|S)ah Find files by glob-matching. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.glob). Args: path (`str`): Path pattern to match. Returns: `List[str]`: List of paths matching the pattern. r)rt)rgr4r<rGglobr8rSrrrNr:r;rs zHfFileSystem.glob)rSrzwithdirsrrr)r/c s,|dk r|dkrtd|j||d}|}z"|j|fd||j|d|Wn^tk rz0|j|fd|i|dd kr|iiniWntk riYnXYnVX|sd d Dn2|j|fd|ji|} | dd kr| gnd dDt} |s| Sfdd| DSdS)a List all files below path. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.find). Args: path (`str`): Root path to list files from. maxdepth (`int`, *optional*): Maximum depth to descend into subdirectories. withdirs (`bool`, *optional*): Include directory paths in the output. Defaults to False. detail (`bool`, *optional*): If True, returns a dict mapping paths to file information. Defaults to False. refresh (`bool`, *optional*): If True, bypass the cache and fetch the latest data. Defaults to False. revision (`str`, *optional*): The git revision to list from. Returns: `Union[List[str], Dict[str, Dict[str, Any]]]`: List of paths or dict of file information. Nrzmaxdepth must be at least 1rtT)ryrr)rzr)rrcSsg|]}|ddkr|qSrr:rr:r:r;r~Es z%HfFileSystem.find..rcSsi|]}|d|qSrr:rr:r:r; Jsz%HfFileSystem.find..csi|]}||qSr:r:)r|rrr:r;rOs) rUrgr<rr)rinfoFileNotFoundErrorsorted) r8rSrzrrrr)rrrkrnamesr:rr;finds>   zHfFileSystem.find)path1path2r)r/c Ks|j||d}|j||d}|j|jko2|j|jk}|rd|d|}|jj|j|j|j|d||ddt|j|j|jdgdnj|j |d |jd} | } W5QRXd|d|}|jj | |j|j|j |j|j|d||dd |j |d |j |d d S) a Copy a file within or between repositories. > [!WARNING] > Note: When possible, use `HfApi.upload_file()` for better performance. Args: path1 (`str`): Source path to copy from. path2 (`str`): Destination path to copy to. revision (`str`, *optional*): The git revision to copy from. rtzCopy z to rurvr0)Zsrc_path_in_repor*Z src_revision)r(r'r)rurvrrmZpath_or_fileobjr*r(rEr'r)rurvrwN)rgr'r(rIrr)r4rr*openread upload_filerErlr<) r8rrr)rrZresolved_path1Zresolved_path2Z same_reporufcontentr:r:r;cp_fileQsD    zHfFileSystem.cp_filecKs"|j|f|ddi}|ddS)ai Get the last modified time of a file. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.modified). Args: path (`str`): Path to the file. Returns: `datetime`: Last commit date of the file. rTrdater)r8rSrrrr:r:r;modifieds zHfFileSystem.modified)rSrr)r/c  s|j|d}||dd}|jsxdddd|rt|jj|j|j|jdd }dt |j |j |j d d nDd| }|s||jkr||||jkrևfd d |j|D} | std| d|sdks|rrddkr|jj|j|j||j|jd} | s*td| d} t|j|j|jd|jd} t| tr| d| j| jd| j| j| j| jdn| d| jdd| j| jd|sfdddDdk stS)a Get information about a file or directory. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.info). > [!WARNING] > Note: When possible, use `HfApi.get_paths_info()` or `HfApi.repo_info()` for better performance. Args: path (`str`): Path to get info for. refresh (`bool`, *optional*): If True, bypass the cache and fetch the latest data. Defaults to False. revision (`str`, *optional*): The git revision to get info from. Returns: `Dict[str, Any]`: Dictionary containing file information (type, size, commit info, etc.). rtrFrrN)rrrr)r'r))oidtitler)rrcsg|]}|dkr|qSrr:rrwr:r;r~s z%HfFileSystem.info..r)rr)r'r0rr2rrrcsi|]}||qSr:r:)r|krr:r;rsz%HfFileSystem.info..)rrr)rgr<r4r*rIZlist_repo_commitsr(r'r)r! commit_idrZ created_atrjrKrraZget_paths_infor&r-rcr"rSrrrrrrAssertionError) r8rSrr)rrrkrrrZout1Z paths_inforrr:)rrSr;rs     &     zHfFileSystem.infocKs>z*|ddr|||j|f|WdSYdSXdS)a Check if a file exists. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.exists). > [!WARNING] > Note: When possible, use `HfApi.file_exists()` for better performance. Args: path (`str`): Path to check. Returns: `bool`: True if file exists, False otherwise. rFTN)r4rlrrr:r:r;existss  zHfFileSystem.existscCs0z||ddkWStk r*YdSXdS)ai Check if a path is a directory. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.isdir). Args: path (`str`): Path to check. Returns: `bool`: True if path is a directory, False otherwise. rrFN)rOSErrorr8rSr:r:r;r{s zHfFileSystem.isdircCs(z||ddkWSYdSXdS)a` Check if a path is a file. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.isfile). Args: path (`str`): Path to check. Returns: `bool`: True if path is a file, False otherwise. rrFNrrr:r:r;isfile&s zHfFileSystem.isfilecCsB||}t|j|j|j|j|jd}||r>|ddd}|S)z Get the HTTP URL of the given path. Args: path (`str`): Path to get URL for. Returns: `str`: HTTP URL to access the file or directory on the Hub. )r'r)rDz /resolve/z/tree/r) rgrr(r*r'r)rDr{replace)r8rSrkurlr:r:r;r8s  zHfFileSystem.urlr.c sV|d}t|dh}t|ttfr6t|dkrRtj||f||d|St |r`|}n| |r|t j |dddSt|t tfrt j t j|ddd}|dkrt|d}d}|} |j||d } |j||d d } || zTtt| j| j| j| j|jd ||| d|jt|tr,|j ndd |!| W5|rP|XdS) aC Copy single remote file to local. > [!WARNING] > Note: When possible, use `HfApi.hf_hub_download()` for better performance. Args: rpath (`str`): Remote path to download from. lpath (`str`): Local path to download to. callback (`Callback`, *optional*): Optional callback to track download progress. Defaults to no callback. outfile (`IO`, *optional*): Optional file-like object to write to. If provided, `lpath` is ignored. r)r)callbackoutfileT)exist_okNFwbrtrr(r)filenamer'rD)r temp_fileZdisplayed_filename expected_sizeZ resume_sizeheadersZ _tqdm_bar)"r4setkeysrcrrrrGget_filerr{rmakedirsrArrSdirnamertellrgrZset_sizecloserrr(r)r*r'rDrI_build_hf_headersZtqdmseek) r8ZrpathZlpathrrrrr)Zunhandled_kwargsZ close_fileZ initial_posZresolve_remote_pathrrNr:r;rOsL    zHfFileSystem.get_filecCs tddS)zA context within which files are committed together upon exit Requires the file class to implement `.commit()` and `.discard()` for the normal and exception cases. (Transactional commits are not supported.NrXr}r:r:r; transactions zHfFileSystem.transactioncCs tddS)z@Begin write transaction for deferring files, non-context versionrNrr}r:r:r;start_transactionszHfFileSystem.start_transactioncCs"tt||j|j|j|jdffS)N)rKrJ) make_instancerZ storage_argsrMrKrJr}r:r:r; __reduce__szHfFileSystem.__reduce__)N)N)rmNN)N)FNN)TFN)FFNFN)NFFFN)N)FN)-r=r>r?r@Z root_markerprotocolr rArboolintrHr ExceptionrRr&rgrlrsrxrr r rrrr rrrrrrrrr{rrrrpropertyrrr __classcell__r:r:rNr;rC:s%  _  / + v* ?8"bD rCcseZdZdeeeedfdd ZfddZeee ddd Z dd d d Z de ddddZ dfdd Zed ddZZS)rnN)fsrSr)c szz|j||d|_WnDtk rX}z&d|ddkrFt|d|W5d}~XYnXtj||jf||dS)Nrtwror0B. Make sure the repository and revision exist before writing data.)rgrkrr4rGrHr<)r8rrSr)rrrQrNr:r;rHszHfFileSystemFile.__init__cst|dsdStSNrkhasattrrG__del__r}rNr:r;rs zHfFileSystemFile.__del__)startendr/cCsjdd|d|di|jj}t|jj|jj|jj|jj|jj d}t d||t j d}t ||jS)Nrangezbytes=-rrGET)rrP)rrIrrrkr(r)r*r'rDr%rHF_HUB_DOWNLOAD_TIMEOUTr$r)r8rrrrrr:r:r; _fetch_ranges zHfFileSystemFile._fetch_ranger.cCstjddd|_dS)Nzhffs-F)prefixdelete)tempfileNamedTemporaryFilerr}r:r:r;_initiate_uploadsz!HfFileSystemFile._initiate_uploadF)finalr/c Cs|jd|j}|j||r|j|jjj|jj |j j |j j |jj |j j|j j|jd|jddt|jj |jj|j ddS)Nrrurvrrw)bufferrrrwriterrrIrrrkr*r(rEr'r)rrr4rremoverlr<)r8rblockr:r:r; _upload_chunks&       zHfFileSystemFile._upload_chunkrc sv|jdkrj|dks|dkrj|jdkrj|jj|jddd,}|}|jt|7_|W5QRSQRXt|S)a)Read remote file. If `length` is not provided or is -1, the entire file is downloaded and read. On POSIX systems and if `hf_transfer` is not enabled, the file is loaded in memory directly. Otherwise, the file is downloaded to a temporary file and read from there. rmNrr)rF)rolocrrrSrrrG)r8lengthrrrNr:r;rs $zHfFileSystemFile.readcCs|j|jSNrrrSr}r:r:r;rszHfFileSystemFile.url)N)F)r)r=r>r?rCrAr rHrrbytesrrrr rrrr:r:rNr;rns rncsveZdZdeeeeeeedfdd Zdeedd d Zded d dZ edddZ fddZ ddZ Z S)rqrmNrnone)rrSror)rF cache_typec s|dkrtd||dkr,td|d|krDtd|dz|j||d|_WnBtk r}z$d|d d krt|d |W5d}~XYnX|jdd |_tj||jf|||d |d|_ |dS)Nrz:HfFileSystemStreamFile only supports block_size=0 but got rz?HfFileSystemStreamFile only supports cache_type='none' but got rz;HfFileSystemStreamFile only supports reading but got mode=''rtror0r)rrrorFr) rUrgrkrr4r<detailsrGrHresponse) r8rrSror)rFrrrrQrNr:r;rHs8 zHfFileSystemStreamFile.__init__)r whencecCs6|dkr|dkrdS||jkr*|dkr*dStddS)NrrzCannot seek streaming HF file)r rU)r8r rr:r:r;rs zHfFileSystemStreamFile.seekr)r c CsZ|dkr|fnd}|jdkrjt|jj|jj|jj|jj|jjd}t d||jj dt j d|_t|jzd|jj_|jjj|}Wntk rB|jt|jj|jj|jj|jj|jjd}t d|dd|ji|jj dt j d|_t|jzd|jj_|jjj|}Wn"tk r<|jYnXYnX|jt|7_|S) Nrr:rrT)rstreamrPRangez bytes=%d-)rrrkr(r)r*r'rrDr%rIrrrr$rawdecode_contentrrrr r)r8r Z read_argsrrr:r:r;r%sZ        zHfFileSystemStreamFile.readr.cCs|j|jSrrr}r:r:r;rVszHfFileSystemStreamFile.urlcst|dsdStSrrr}rNr:r;rYs zHfFileSystemStreamFile.__del__cCst|j|j|j|j|jjffSr)reopenrrSro blocksizecacherr}r:r:r;r_sz!HfFileSystemStreamFile.__reduce__)rmNrr)r)r)r=r>r?rCrAr rrHrrrrrrr:r:rNr;rqs"1 rq)r)r/cCst|r|St|Sr)r\rd safe_quotertr:r:r;r7csr7)sr/cCs t|ddS)Nr0)safe)r)r r:r:r;rgsr)rSrer/cCsR|}t|tr|d}n*t|tr0|d}nt|trD|d}t||dS)Nz (repository not found)z (revision not found)z (invalid repository id))rcrrr#r)rSremsgr:r:r;raks      rarrSrorFrcCs|j||||dS)Nr)rr#r:r:r;rvsrcCs,|||}|D]\}}t|||q|Sr)itemssetattr)clsrLrrZinstance_cache_attributes_dictrattrZ cached_valuer:r:r;rzs r)Hrrer collectionsr dataclassesrrr itertoolsrpathlibrtypingrr r r r r rr urllib.parserrZfsspecZfsspec.callbacksrrrZ fsspec.utilsrrequestsrr0rZ _commit_apirrerrorsrrrZ file_downloadrrZhf_apir r!r"utilsr#r$r%compileVERBOSEr\r&ZAbstractFileSystemrCspecZAbstractBufferedFilernrqrAr7rrrarrrr:r:r:r;sN    (    }Me