U
    ©Ê¹jž9  ã                   @   s  U d dl Z d dlZd dlZd dlZd dlmZ d dlmZ d dlm	Z	m
Z
mZmZmZmZ d dlZd dlmZ d dlmZ d dlmZmZmZmZmZmZmZ d dlmZmZmZm Z m!Z!m"Z"m#Z# d d	l$m%Z%m&Z&m'Z' d d
l(m)Z)m*Z* d dl+m,Z, d dl-m.Z. d dl/m0Z0 d dl1m2Z2 e 3e4¡Z5ej6e7d< ddddddgZ8G dd„ deƒZ9G dd„ deƒZ:ee;ef e!edœdd„Z<e	e e	e dœdd„Z=ee;ef e>edœdd„Z?d-e	e e>e
e	e e!f dœd d„Z@e"e!d!œd"d#„ZAeee>d$œd%d&„ZBejCee>d'œd(d)„ZDe	e e!e>d*œd+d,„ZEdS ).é    N)ÚChainMap)Úreduce)ÚListÚTupleÚDictÚAnyÚUnionÚcast)Únarrow_tensor_by_index)ÚDTensor)ÚSavePlannerÚLoadPlannerÚSavePlanÚLoadPlanÚReadItemÚ	WriteItemÚWriteItemType)ÚBytesStorageMetadataÚChunkStorageMetadataÚTensorStorageMetadataÚMetadataIndexÚMetadataÚSTATE_DICT_TYPEÚSTORAGE_TYPES)Ú_create_read_itemsÚ_create_write_itemsÚ"_create_default_metadata_only_plan)ÚFLATTEN_MAPPINGÚflatten_state_dict)Ú_flatten_sharded_tensors)Údedup_tensors)Úfind_state_dict_object)Úset_elementÚloggerÚDefaultSavePlannerÚDefaultLoadPlannerÚcreate_default_local_load_planÚcreate_default_global_load_planÚcreate_default_local_save_planÚcreate_default_global_save_planc                   @   s¸   e Zd ZU eed< deeeddœdd„Zeeddœdd	„Ze	d
œdd„Z
ee	 eee	 ef dœdd„Ze	e	dœdd„Zeeejejf dœdd„Zeedœdd„Zeedœdd„ZdS )r$   ÚmappingsTN)r   Úflatten_sharded_tensorsÚdedup_replicated_tensorsÚreturnc                 C   s   || _ || _|| _i | _d S ©N)r   r+   r,   r*   )Úselfr   r+   r,   © r0   úk/var/www/html/bwave/python/venv/lib/python3.8/site-packages/torch/distributed/checkpoint/default_planner.pyÚ__init__G   s    zDefaultSavePlanner.__init__©Ú
state_dictÚis_coordinatorr-   c                 C   s2   | j rt |ƒ\}| _| jr"t|ƒ}|| _|| _d S r.   )r   r*   r+   r   r4   r5   )r/   r4   r5   r0   r0   r1   Úset_up_plannerR   s    z!DefaultSavePlanner.set_up_planner©r-   c                 C   s0   t | j| jƒ}| jr$tj|| jd�}|| _| jS )N©Zplanner_data)r(   r4   r5   r   ÚdataclassesÚreplacer*   Úplan)r/   r;   r0   r0   r1   Úcreate_local_plan\   s     ÿz$DefaultSavePlanner.create_local_plan©Ú	all_plansr-   c                 C   sr   | j rt|ƒ}t|ƒ\}}| jrHdd„ |D ƒ}tt|Ž ƒ}tj||d�}t||ƒsZt	dƒ‚|| _
|| _| j
| jfS )Nc                 S   s   g | ]
}|j ‘qS r0   r8   )Ú.0Úpr0   r0   r1   Ú
<listcomp>s   s     z9DefaultSavePlanner.create_global_plan.<locals>.<listcomp>r8   zFailed to validate global plan)r,   r    r)   r   Údictr   r9   r:   Ú_validate_global_planÚ
ValueErrorÚglobal_planÚmetadata)r/   r>   rE   rF   Zplanner_data_dictZmerged_mappingsr0   r0   r1   Úcreate_global_planf   s     ÿ
z%DefaultSavePlanner.create_global_plan©Únew_planr-   c                 C   s
   || _ |S r.   )r;   ©r/   rI   r0   r0   r1   Úfinish_plan�   s    zDefaultSavePlanner.finish_plan)Ú
write_itemr-   c                 C   s   |   |j¡}|  ||¡S r.   )Úlookup_objectÚindexÚtransform_object)r/   rL   Úobjectr0   r0   r1   Úresolve_data…   s    zDefaultSavePlanner.resolve_data©rN   r-   c                 C   s   t | j|ƒS ©zo
        This is an extension from the planner interface to make it easy to extend the default planner
        ©r!   r4   ©r/   rN   r0   r0   r1   rM   ‹   s    z DefaultSavePlanner.lookup_object)rL   rP   c                 C   s(   |j tjkr$t ¡ }t ||¡ |}|S rS   )Útyper   ÚBYTE_IOÚioÚBytesIOÚtorchÚsave)r/   rL   rP   Úbytesr0   r0   r1   rO   ‘   s
    z#DefaultSavePlanner.transform_object)TTT)Ú__name__Ú
__module__Ú__qualname__r   Ú__annotations__Úboolr2   r   r6   r   r<   r   r   r   rG   rK   r   r   rZ   ÚTensorrX   rY   rQ   r   r   rM   rO   r0   r0   r0   r1   r$   D   s0   
   üû þ
þþc                   @   sÚ   e Zd ZU dZeed< eed< d$eeddœdd„Zee	edd	œd
d„Z
edœdd„Zee ee dœdd„Zeedœdd„Zeejddœdd„Zedœdd„Zeejddœdd„Zeejdœdd „Zeejd!œd"d#„ZdS )%r%   zð
    DefaultLoadPlanner that adds multiple features on top of LoadPlanner.

    In particular it adds the following:

    flatten_state_dict: Handle state_dict with nested dicts
    flatten_sharded_tensors: For FSDP in 2D parallel mode
    Úoriginal_state_dictr*   TN)r   r+   r-   c                 C   s   || _ || _i | _i | _d S r.   )r   r+   rc   r*   )r/   r   r+   r0   r0   r1   r2   ©   s    zDefaultLoadPlanner.__init__)r4   rF   r5   r-   c                 C   s>   || _ | jrt|ƒ}| jr(t|ƒ\}| _|| _|| _|| _d S r.   )rc   r+   r   r   r*   r4   rF   r5   )r/   r4   rF   r5   r0   r0   r1   r6   ³   s    z!DefaultLoadPlanner.set_up_plannerr7   c                 C   s   t | j| jƒS r.   )r&   r4   rF   )r/   r0   r0   r1   r<   Å   s    z$DefaultLoadPlanner.create_local_plan)rE   r-   c                 C   s   t |ƒS r.   )r'   )r/   rE   r0   r0   r1   rG   È   s    z%DefaultLoadPlanner.create_global_planrH   c                 C   s   |S r.   r0   rJ   r0   r0   r1   rK   Ë   s    zDefaultLoadPlanner.finish_plan)Ú	read_itemÚvaluer-   c                 C   s>   | j r&t| j| j|jj t |¡ƒ nt |¡| j|jj< d S r.   )	r   r"   rc   r*   Ú
dest_indexÚfqnrZ   Úloadr4   )r/   rd   re   r0   r0   r1   Ú
load_bytesÎ   s    ýzDefaultLoadPlanner.load_bytes)rd   c                 C   s   |   |j¡}|  ||¡S r.   )Úlookup_tensorrf   Útransform_tensor©r/   rd   Útensorr0   r0   r1   Úresolve_tensorØ   s    z!DefaultLoadPlanner.resolve_tensor)rd   rm   r-   c                 C   s   d S r.   r0   rl   r0   r0   r1   Úcommit_tensorÜ   s    z DefaultLoadPlanner.commit_tensorrR   c                 C   s   t | j|ƒS rS   rT   rU   r0   r0   r1   rj   ß   s    z DefaultLoadPlanner.lookup_tensor)rd   rm   c                 C   s   t ||j|jƒS rS   )r
   Zdest_offsetsÚlengthsrl   r0   r0   r1   rk   å   s
      ÿz#DefaultLoadPlanner.transform_tensor)TT)r]   r^   r_   Ú__doc__r   r`   r   ra   r2   r   r6   r   r<   r   rG   rK   r   rX   rY   ri   rn   rZ   rb   ro   r   rj   rk   r0   r0   r0   r1   r%   œ   s.   
	  ýüû
)r4   rF   r-   c                 C   sb   g }|   ¡ D ]L\}}|j| }t|tƒrH|j ¡ d k	rX|t|||ƒ7 }q|t|||ƒ7 }qt|ƒS r.   )ÚitemsÚstate_dict_metadataÚ
isinstancer   Údevice_meshÚget_coordinater   r   )r4   rF   Úrequestsrg   ÚobjÚmdr0   r0   r1   r&   î   s    

r=   c                 C   s   | S )z¼
    Create global load plan used by DefaultLoadPlanner.

    The default load behavior involved no global coordination and this function
    currently doesn't change the local plans.
    r0   )r>   r0   r0   r1   r'   
  s    	r3   c                 C   sd   g }|   ¡ D ]N\}}t|tƒr<|j ¡ dk	rZ|t||ƒ7 }qt|tjƒsL|r|t||ƒ7 }qt|ƒS )a  
    Create the ``SavePlan`` used by DefaultSavePlanner.

    On non-coordinator ranks, this function ignores tensors and non-tensor objects,
    only producing writes for ShardedTensor objects.

    On the coordinator rank, produce writes for all values.
    N)	rr   rt   r   ru   rv   r   rZ   rb   r   )r4   r5   rw   rg   rx   r0   r0   r1   r(     s    
T)r>   Úrewrite_index_hintsr-   c           
      C   s"  i }g }| D �]}g }|j D ]â}|jtjks<|jj|ks<t‚|jtjkrbtƒ ||jj< | 	|¡ q|j
dk	spt‚tt| |jjt|j
j|j
jg d�¡ƒ}|}|rÄtj|jt|jƒd�}	tj||	d�}| 	|¡ |j
jdk	sîtd|jj› d�ƒ‚|j 	|j
j¡ q| 	tj||d�¡ q|t|ƒfS )a6  
    Create the global plan and metadata used by DefaultSavePlanner.

    Metadata is produced by concatenating the metadata of all ``WriteItem`` from the supplied plans.

    The only global planning change is to update index hints in all ``MetadataIndex`` objects if
    ``rewrite_index_hints`` is True.
    N)Ú
propertiesÚsizeÚchunks)rN   zZ
                    Cannot create MD for tensor without bounds.
                    FQN: z
                )rr   )rr   rV   r   ZSHARDrN   rg   ÚAssertionErrorrW   r   ÚappendZtensor_datar	   r   Ú
setdefaultr{   r|   r9   r:   Úlenr}   Úchunkr   )
r>   rz   ry   Z	new_plansr;   Z	new_itemsÚitemZ	tensor_mdZnew_itemZ	new_indexr0   r0   r1   r)   .  sN    

ýþþ ÿ

ÿþþ)r4   r-   c                 C   s   t | ƒ}t|gƒ\}}|S )z^
    Return the ``Metadata`` if DefaultSavePlanner was used to checkpoint ``state_dict``.
    )r   r)   )r4   r;   Ú_ry   r0   r0   r1   Ú_create_default_local_metadatae  s    r…   )Úbox0Úbox1r-   c                 C   sd   t | jƒ}t|ƒD ]L}| j| |j| |j|  kr: dS |j| | j| | j|  kr dS qdS )zC
    Checks if two boxes overlap. Tuples are (offset, lengths)
    FT)r�   ÚoffsetsÚrangeÚsizes)r†   r‡   ZndimsÚir0   r0   r1   Ú_check_box_overlapn  s    
rŒ   )Úouter_box_sizeÚ	inner_boxr-   c                 C   s`   t t| ƒƒD ]N}|j| dk r$ dS |j| dk r8 dS |j| |j|  | | kr dS qdS )Nr   FT)r‰   r�   rˆ   rŠ   )r�   rŽ   r‹   r0   r0   r1   Ú_check_box_boundsƒ  s    r�   )rE   rF   r-   c           
   	   C   sè   d}|j  ¡ D ]Ô\}}t|tƒr"qt|jƒdkr2qd}t|jƒD ]t\}}t|j|ƒsjt	 
d||j|¡ d}|ttj|jdƒ7 }|j|d d … D ]"}t||ƒr�t	 
d|||¡ d}q�q@ttj|jdƒ}	||	krt	 
d||	|¡ d}q|S )NTr   z~
                        key:%s has out of bounds chunk:
                        tensor-size:%s chunk: %s
                    Fé   z$key:%s has overlapping chunks: %s %szq
                    key:%s invalid fill tensor-volume:
                    %s chunks-volume: %s
                )rs   rr   rt   r   r�   r|   Ú	enumerater}   r�   r#   Úwarningr   ÚoperatorÚmulrŠ   rŒ   )
rE   rF   Úall_goodÚkeyre   Zchunks_volumeZ	chunk_idxZchunk0Zchunk1Ztensor_volumer0   r0   r1   rC   ‘  sH    
  ü
   ÿ  ürC   )T)Fr9   rX   Úloggingr“   Úcollectionsr   Ú	functoolsr   Útypingr   r   r   r   r   r	   rZ   Ztorch.distributed._shard._utilsr
   Ztorch.distributed._tensorr   Z$torch.distributed.checkpoint.plannerr   r   r   r   r   r   r   Z%torch.distributed.checkpoint.metadatar   r   r   r   r   r   r   Z,torch.distributed.checkpoint.planner_helpersr   r   r   Z)torch.distributed.checkpoint._nested_dictr   r   Z2torch.distributed.checkpoint._sharded_tensor_utilsr   Z+torch.distributed.checkpoint._dedup_tensorsr    Z"torch.distributed.checkpoint.utilsr!   Z&torch.distributed.checkpoint._traverser"   Ú	getLoggerÚ__file__r#   ÚLoggerr`   Ú__all__r$   r%   Ústrr&   r'   ra   r(   r)   r…   rŒ   ÚSizer�   rC   r0   r0   r0   r1   Ú<module>   st     $
$
úXS
ýþ
 þ þý7
 þ þ þ