U ʹj<#@sUddlmZmZddlZddlmZddlmZddlm Z ddl m Z ddl m Z mZdd lmZmZmZmZmZmZdd lmZmZmZmZmZmZdd lmZmZd gZ ee!e"d <ej#edddZ$eedddZ%eeedddZ&e!e edddZ'e!eeedddZ(e!ej#edddZ)e!edd d!Z*d"d#Z+d$d%Z,e!eeeeed&d'd Z-eed(d)d*Z.e!eeed+d,d-Z/e edd.d/Z0e!eeeed0d1d2Z1dS)3)AnyListN) ShardMetadata) ShardedTensor)TensorProperties)DTensor)compute_local_shapecompute_local_offset) LoadItemTypeSavePlanReadItem WriteItem WriteItemTypeTensorWriteData)BytesStorageMetadataChunkStorageMetadataTensorStorageMetadata MetadataIndexSTATE_DICT_TYPE STORAGE_TYPES)+_shards_get_overlap_region_wrt_saved_tensor"_check_shard_metadata_pair_overlap create_read_items_for_chunk_list__all__)tensorreturncCs$ttdgt||dS)Nroffsetssizes)rtorchSizelensize)rr$k/var/www/html/bwave/python/venv/lib/python3.8/site-packages/torch/distributed/checkpoint/planner_helpers.py_create_chunk_from_tensor%sr&)shard_mdrcCstt|jt|jdS)Nr)rr r! shard_offsetsZ shard_sizes)r'r$r$r%_chunk_for_shard,s  r))sharded_tensorr'rcCstt||j|jdS)Nchunk propertiesr#)rr)metadataZtensor_propertiesr#)r*r'r$r$r%_sharded_tensor_metadata3s r/)fqnrrc Csz|j}|jdkstdtt|j||j}tt|j||j}t t ||t j t t||dt||ddS)Nr .Only 1D DeviceMeshes can currently be handled.rr+indextypeZ tensor_data) device_meshndimAssertionErrorr r!rshape placementsr rrrSHARDrrrcreate_from_tensorZto_localr#)r0rr5rrr$r$r%_create_write_items_for_dtensor=s& r<)r0r*r'rcCs(t|j}tt||tjt||dS)Nr2)r r!r(rrrr:r/)r0r*r'rr$r$r%_create_write_item_for_shardUs  r=cCsNtdgt|}tt||tjtt ||dt ||ddS)Nrrr+r2) r r!r"r#rrrTENSORrrrr;)r0rrr$r$r%_create_write_item_for_tensor`sr?r0bytescCstt|tjdS)N)r3r4)rrrBYTE_IOr@r$r$r%_create_write_item_for_bytesiomsrCc Cs.ttj|t|f|t|ft|fdSN)r4 dest_index dest_offsets storage_indexstorage_offsetslengths)r r rBr r!rEZ dest_offsetrGZstorage_offsetlengthr$r$r%_create_read_item_for_byteiots   rLc Cs(ttj|t||t|t|dSrD)r r r>r r!rErFrGrHrIr$r$r%_create_read_item_for_tensorsrN)r0 checkpoint_md local_chunksrc Csg}t|D]\}}t|jD]\}}t||s2qg}g} g} t||dD]*\} } } }|| | | | |qJ|tt||j|| t||j||| dqq |S)aX Creates a list of ``ReadItem`` based on the checkpoint and local chunks. This applies the resharding algorithm and computes the reads needed to satisfy ``local_chunks`` with a checkpoint described by ``checkpoint_md``. Args: fqn (str) : The state_dict FQN to pass to ``ReadItem``. checkpoint_md (TensorStorageMetadata): metadata for a given tensor from a checkpoint. local_chunks (List[ChunkStorageMetadata]): Local chunks that needs to be loaded. Returns: A list of ``ReadItem`` that will satisfy all input chunks. )Z saved_shardZ current_shardrM) enumeratechunksrrappendrNrr)r0rOrPZ read_itemsidxshardZ storage_idxZ storage_mdrHrFrIdimZoffset_for_saved_tensorZoffset_for_current_tensorrKr$r$r%rsR    ) state_dictrcCsg}|D]\}}t|tr0|t||q t|tr^|jD]}|t|||qDq t|t j r||t ||q |t ||q t |S)N)items isinstancerrSr<rr.Zshards_metadatar=r Tensorr?rCr )rWrequestsr0objr'r$r$r%"_create_default_metadata_only_plans    r])r0objectrcs`ttrtgSttr8fddDSttjrPtgStgSdS)Ncsg|]}t|jqSr$)r=r..0rUr0r^r$r% sz'_create_write_items..) rYrr<r local_shardsr rZr?rCrar$rar%_create_write_itemss      rdcCsP|j}|jdkstdtt|j||j}tt|j||j}t ||dS)Nr r1r) r5r6r7r r!rr8r9r r)rr5rrr$r$r%_create_chunk_from_dtensorsre)r0mdr\rcCst|tst|tr t|g}nTt|tr>dd|D}n6t|tjrVt|g}nt d|ddt |t |||St t |dt |dddgSdS)NcSsg|]}t|jqSr$)r)r.r_r$r$r%rbsz&_create_read_items..z Invalid checkpoint metadata for z, z(expected BytesStorageMetadata but found rrJ)rYrrrerrcr rZr& ValueErrorr4rrLr)r0rfr\rPr$r$r%_create_read_itemss*         rh)2typingrrr Z!torch.distributed._shard.metadatarZ'torch.distributed._shard.sharded_tensorrZ0torch.distributed._shard.sharded_tensor.metadatarZtorch.distributed._tensorrZ torch.distributed._tensor._utilsrr Zplannerr r r rrrr.rrrrrrZ reshardingrrrstr__annotations__rZr&r)r/r<r=r?rCrLrNrr]rdrerhr$r$r$r%sH           =