U ʹj@<@sddlZddlZddlZddlZddlZddlZddlmZddlm Z m Z ddl m Z ddl mZmZmZmZmZmZddlmZmZmZeeZdd d d hZd d ddhZe GdddZeejefeeefdddZ eeejfeeeeee e!ffddddZ"eeejfeeejfddddZ#ej$eeeeje!fddddZ%eee e!fe!ddd Z&eed!d"d#Z'eeedd$d%d&Z(ej$ej)e*d'd(d)Z+dS)*N)contextmanager) dataclassfield)Path)AnyDict GeneratorIterableTupleUnion)DDUFCorruptedFileErrorDDUFExportErrorDDUFInvalidEntryNameErrorz.jsonz.modelz .safetensorsz.txtz config.jsonztokenizer_config.jsonzpreprocessor_config.jsonzscheduler_config.jsonc@sjeZdZUdZeed<eed<eed<eddZe ed<e e e ddfd d d Z deed ddZdS) DDUFEntryaObject representing a file entry in a DDUF file. See [`read_dduf_file`] for how to read a DDUF file. Attributes: filename (str): The name of the file in the DDUF archive. offset (int): The offset of the file in the DDUF archive. length (int): The length of the file in the DDUF archive. dduf_path (str): The path to the DDUF archive (for internal use). filenamelengthoffsetF)repr dduf_pathNreturnc csV|jd@}tj|dtjd}||j|j|jVW5QRXW5QRXdS)a-Open the file as a memory-mapped file. Useful to load safetensors directly from the file. Example: ```py >>> import safetensors.torch >>> with entry.as_mmap() as mm: ... tensors = safetensors.torch.load(mm) ``` rbr)raccessN)ropenmmapfileno ACCESS_READrr)selffmmr!b/var/www/html/bwave/python/venv/lib/python3.8/site-packages/huggingface_hub/serialization/_dduf.pyas_mmap9s zDDUFEntry.as_mmaputf-8)encodingrc CsD|jd.}||j||jj|dW5QRSQRXdS)zRead the file as text. Useful for '.txt' and '.json' entries. Example: ```py >>> import json >>> index = json.loads(entry.read_text()) ``` r)r%N)rrseekrreadrdecode)rr%rr!r!r" read_textJs  zDDUFEntry.read_text)r$)__name__ __module__ __qualname____doc__str__annotations__intrrrrrbytesr#r)r!r!r!r"r"s r)rrc Cs$i}t|}td|tt|d}|D]}td|j|j tj kr`t dzt |jWn4t k r}zt d|j|W5d}~XYnXt||}t|j||j|d||j<q6W5QRXd|krt d t|d}t||td |d t|d |S) a Read a DDUF file and return a dictionary of entries. Only the metadata is read, the data is not loaded in memory. Args: dduf_path (`str` or `os.PathLike`): The path to the DDUF file to read. Returns: `Dict[str, DDUFEntry]`: A dictionary of [`DDUFEntry`] indexed by filename. Raises: - [`DDUFCorruptedFileError`]: If the DDUF file is corrupted (i.e. doesn't follow the DDUF format). Example: ```python >>> import json >>> import safetensors.torch >>> from huggingface_hub import read_dduf_file # Read DDUF metadata >>> dduf_entries = read_dduf_file("FLUX.1-dev.dduf") # Returns a mapping filename <> DDUFEntry >>> dduf_entries["model_index.json"] DDUFEntry(filename='model_index.json', offset=66, length=587) # Load model index as JSON >>> json.loads(dduf_entries["model_index.json"].read_text()) {'_class_name': 'FluxPipeline', '_diffusers_version': '0.32.0.dev0', '_name_or_path': 'black-forest-labs/FLUX.1-dev', ... # Load VAE weights using safetensors >>> with dduf_entries["vae/diffusion_pytorch_model.safetensors"].as_mmap() as mm: ... state_dict = safetensors.torch.load(mm) ``` zReading DDUF file rzReading entry z)Data must not be compressed in DDUF file.z!Invalid entry name in DDUF file: N)rrrrmodel_index.json7Missing required 'model_index.json' entry in DDUF file.zDone reading DDUF file z. Found z entries)rloggerinfozipfileZipFiler.infolistdebugr compress_type ZIP_STOREDr _validate_dduf_entry_namer_get_data_offsetr file_sizejsonloadsr)_validate_dduf_structurekeyslen)rentrieszfr6erindexr!r!r"read_dduf_fileZs2'  $ rI)rrErc Csztd|dt}d}tt|dtj}|D]\}}||krTtd||||dkrzt t | }Wn.t j k r}ztd|W5d}~XYnXz t|}Wn2tk r}ztd||W5d}~XYnXtd |d t|||q6W5QRX|dkr(td zt||Wn.tk rd}ztd |W5d}~XYnXtd |dS)a Write a DDUF file from an iterable of entries. This is a lower-level helper than [`export_folder_as_dduf`] that allows more flexibility when serializing data. In particular, you don't need to save the data on disk before exporting it in the DDUF file. Args: dduf_path (`str` or `os.PathLike`): The path to the DDUF file to write. entries (`Iterable[Tuple[str, Union[str, Path, bytes]]]`): An iterable of entries to write in the DDUF file. Each entry is a tuple with the filename and the content. The filename should be the path to the file in the DDUF archive. The content can be a string or a pathlib.Path representing a path to a file on the local disk or directly the content as bytes. Raises: - [`DDUFExportError`]: If anything goes wrong during the export (e.g. invalid entry name, missing 'model_index.json', etc.). Example: ```python # Export specific files from the local disk. >>> from huggingface_hub import export_entries_as_dduf >>> export_entries_as_dduf( ... dduf_path="stable-diffusion-v1-4-FP16.dduf", ... entries=[ # List entries to add to the DDUF file (here, only FP16 weights) ... ("model_index.json", "path/to/model_index.json"), ... ("vae/config.json", "path/to/vae/config.json"), ... ("vae/diffusion_pytorch_model.fp16.safetensors", "path/to/vae/diffusion_pytorch_model.fp16.safetensors"), ... ("text_encoder/config.json", "path/to/text_encoder/config.json"), ... ("text_encoder/model.fp16.safetensors", "path/to/text_encoder/model.fp16.safetensors"), ... # ... add more entries here ... ] ... ) ``` ```python # Export state_dicts one by one from a loaded pipeline >>> from diffusers import DiffusionPipeline >>> from typing import Generator, Tuple >>> import safetensors.torch >>> from huggingface_hub import export_entries_as_dduf >>> pipe = DiffusionPipeline.from_pretrained("CompVis/stable-diffusion-v1-4") ... # ... do some work with the pipeline >>> def as_entries(pipe: DiffusionPipeline) -> Generator[Tuple[str, bytes], None, None]: ... # Build an generator that yields the entries to add to the DDUF file. ... # The first element of the tuple is the filename in the DDUF archive (must use UNIX separator!). The second element is the content of the file. ... # Entries will be evaluated lazily when the DDUF file is created (only 1 entry is loaded in memory at a time) ... yield "vae/config.json", pipe.vae.to_json_string().encode() ... yield "vae/diffusion_pytorch_model.safetensors", safetensors.torch.save(pipe.vae.state_dict()) ... yield "text_encoder/config.json", pipe.text_encoder.config.to_json_string().encode() ... yield "text_encoder/model.safetensors", safetensors.torch.save(pipe.text_encoder.state_dict()) ... # ... add more entries here >>> export_entries_as_dduf(dduf_path="stable-diffusion-v1-4.dduf", entries=as_entries(pipe)) ``` zExporting DDUF file ''NwzCan't add duplicate entry: r3z#Failed to parse 'model_index.json'.zInvalid entry name: zAdding entry 'z' to DDUF filer4zInvalid DDUF file structure.zDone writing DDUF file )r5r6setr7r8r.r<raddr@rA _load_contentr(JSONDecodeErrorr=rr:_dump_content_in_archiverBr )rrE filenamesrHarchivercontentrGr!r!r"export_entries_as_ddufs4:   " rT)r folder_pathrcs6tttttfdfdd }t||dS)a Export a folder as a DDUF file. AUses [`export_entries_as_dduf`] under the hood. Args: dduf_path (`str` or `os.PathLike`): The path to the DDUF file to write. folder_path (`str` or `os.PathLike`): The path to the folder containing the diffusion model. Example: ```python >>> from huggingface_hub import export_folder_as_dduf >>> export_folder_as_dduf(dduf_path="FLUX.1-dev.dduf", folder_path="path/to/FLUX.1-dev") ``` rc3sztdD]f}|sq|jtkr:td|dq|}t|j dkrftd|dq| |fVqdS)Nz**/*zSkipping file 'z' (file type not allowed)z"' (nested directories not allowed)) rglobis_filesuffixDDUF_ALLOWED_ENTRIESr5r: relative_torDpartsas_posix)pathZpath_in_archiverUr!r"_iterate_over_folders  z3export_folder_as_dduf.._iterate_over_folderN)rr r r.rT)rrUr`r!r_r"export_folder_as_ddufs ra)rRrrSrc Cs|j|dddh}t|ttfrNt|}|d}t||dW5QRXn&t|trd||ntd|dW5QRXdS)NrKT) force_zip64rizInvalid content type for z. Must be str, Path or bytes.) r isinstancer.rshutil copyfileobjr1writer)rRrrSZ archive_fhZ content_pathZ content_fhr!r!r"rPs   rP)rSrcCs@t|ttfrt|St|tr(|Stdt|ddS)zoLoad the content of an entry as bytes. Used only for small checks (not to dump content into archive). z6Invalid content type. Must be str, Path or bytes. Got .N)rcr.r read_bytesr1rtype)rSr!r!r"rN*s   rN) entry_namercCshd|ddtkr$td|d|krInvalid 'model_index.json' content. Must be a dictionary. Got rgcSs"h|]}d|kr|ddqS)rmr)ro).0entryr!r!r" Wsz+_validate_dduf_structure..zMissing required entry 'z' in 'model_index.json'.c3s |]}d|kVqdS)rmNr!)rsZrequired_entryrrfolderr!r" [sz+_validate_dduf_structure..z!Missing required file in folder 'z!'. Must contains at least one of N)rcdictr rianyDDUF_FOLDER_REQUIRED_ENTRIES)rHrrZ dduf_foldersr!rvr"rBBs rB)rFr6rcCs|jdkrtd|j}|j||jd}t|dkrDtdt|ddd}t|ddd}|d||}|S)a1 Calculate the data offset for a file in a ZIP archive. Args: zf (`zipfile.ZipFile`): The opened ZIP file. Must be opened in read mode. info (`zipfile.ZipInfo`): The file info. Returns: int: The offset of the file data in the ZIP archive. Nz+ZipFile object must be opened in read mode.zIncomplete local file header.little)fpr header_offsetr&r'rDr0 from_bytes)rFr6rZlocal_file_headerZ filename_lenZextra_field_lenZ data_offsetr!r!r"r>as    r>),r@loggingrosrdr7 contextlibr dataclassesrrpathlibrtypingrrrr r r errorsr rr getLoggerr*r5rZr{rPathLiker.rIr1rTrar8rPrNr=rBZipInfor0r>r!r!r!r"sD      7"F  [&$"