U ʹjB@sddlZddlZddlZddlmZddlmZmZmZm Z m Z m Z m Z m Z mZddlZddlmZmZmZmZddlmZe dZejfeeeeedd d Zdejeeeejd d d ZGdddZeedddZGdddZdS)NFraction) AnyCallablecastDictListOptionalTupleTypeVarUnion)_probe_video_from_file_read_video_from_file read_videoread_video_timestamps)tqdmT)pts timebase_from timebase_to round_funcreturncCst|d||}||S)zconvert pts between different time bases Args: pts: presentation timestamp, float timebase_from: original timebase. Fraction timebase_to: new timebase. Fraction round_func: rounding function. rr)rrrrZnew_ptsr_/var/www/html/bwave/python/venv/lib/python3.8/site-packages/torchvision/datasets/video_utils.py pts_convertsr)tensorsizestepdilationrcCs|dkrtd||d}|}||||f}|||dd|d|f}|ddkrtd|f}t|||S)a similar to tensor.unfold, but with the dilation and specialized for 1d tensors Returns all consecutive windows of `size` elements, with `step` between windows. The distance between each element in a window is given by `dilation`. rz*tensor should have 1 dimension instead of r)dim ValueErrorZstridenumeltorchZ as_strided)rrrrZo_strider"Z new_stridenew_sizerrrunfolds    r%c@sReZdZdZeeddddZedddZee eee e fd d d Z dS) _VideoTimestampsDatasetz Dataset used to parallelize the reading of the timestamps of a list of videos, given their paths in the filesystem. Used in VideoClips and defined at top level, so it can be pickled when forking. N) video_pathsrcCs ||_dSNr')selfr'rrr__init__8sz _VideoTimestampsDataset.__init__rcCs t|jSr(lenr'r*rrr__len__;sz_VideoTimestampsDataset.__len__idxrcCst|j|Sr()rr')r*r2rrr __getitem__>sz#_VideoTimestampsDataset.__getitem__) __name__ __module__ __qualname____doc__rstrr+intr0r r floatr3rrrrr&/sr&)xrcCs|S)zH Dummy collate function to be used with _VideoTimestampsDataset r)r;rrr _collate_fnBsr<c@seZdZdZd.eeeeeeeeee feeeeeeeedddd Z dd d d Z eee fdd ddZ e eee fd ddZeeddddZed/ejeeeeeeejeeeejffdddZd0eeeeddddZed ddZed ddZed dd Zeeeefd!d"d#Zeeeeeeejfd$d%d&Zeeejejeee fefd!d'd(Zeee fd d)d*Zeee fdd+d,d-ZdS)1 VideoClipsaC Given a list of video files, computes all consecutive subvideos of size `clip_length_in_frames`, where the distance between each subvideo in the same video is defined by `frames_between_clips`. If `frame_rate` is specified, it will also resample all the videos to have the same frame rate, and the clips will refer to this frame rate. Creating this instance the first time is time-consuming, as it needs to decode all the videos in `video_paths`. It is recommended that you cache the results after instantiation of the class. Recreating the clips for different clip lengths is fast, and can be done with the `compute_clips` method. Args: video_paths (List[str]): paths to the video files clip_length_in_frames (int): size of a clip in number of frames frames_between_clips (int): step (in frames) between each clip frame_rate (int, optional): if specified, it will resample the video so that it has `frame_rate`, and then the clips will be defined on the resampled video num_workers (int): how many subprocesses to use for data loading. 0 means that the data will be loaded in the main process. (default: 0) output_format (str): The format of the output video tensors. Can be either "THWC" (default) or "TCHW". rNrTHWC)r'clip_length_in_framesframes_between_clips frame_rate_precomputed_metadata num_workers _video_width _video_height_video_min_dimension_video_max_dimension_audio_samples_audio_channels output_formatrcCs||_||_||_||_| |_| |_| |_| |_| |_ |j dkrTt d| d|dkrf| n | || |||dS)N)r?TCHWz5output_format should be either 'THWC' or 'TCHW', got .)r'rDrErFrGrHrIrJupperrKr!_compute_frame_pts_init_from_metadata compute_clips)r*r'r@rArBrCrDrErFrGrHrIrJrKrrrr+ds    zVideoClips.__init__r,c sg|_g|_ddljjjt|jd|jt d}t t |dT}|D]H}| dt t|\}}fdd|D}|j||j|qFW5QRXdS)Nrr>)Z batch_sizerDZ collate_fn)totalrcsg|]}j|jdqS)Zdtype) as_tensorlong).0cr#rr sz1VideoClips._compute_frame_pts..) video_pts video_fpsZtorch.utils.datautilsdataZ DataLoaderr&r'rDr<rr.updatelistzipextend)r*dlZpbarbatchclipsfpsrrXrrOs   zVideoClips._compute_frame_pts)metadatarcCsV|d|_t|jt|dks$t|d|_t|jt|dksHt|d|_dS)Nr'rZr[)r'r.AssertionErrorrZr[)r*rfrrrrPs   zVideoClips._init_from_metadatacCs|j|j|jd}|S)Nr'rZr[rh)r* _metadatarrrrfs zVideoClips.metadata)indicesrcs~fdd|D}fdd|D}fdd|D}|||d}t|jjj|jjjjjj j j d S)Ncsg|]}j|qSrr)rVir/rrrYsz%VideoClips.subset..csg|]}j|qSr)rZrkr/rrrYscsg|]}j|qSr)r[rkr/rrrYsrh) r@rArBrCrDrErFrGrHrIrJrK) type num_framesrrBrDrErFrGrHrIrJrK)r*rjr'rZr[rfrr/rsubsets,zVideoClips.subset)rZrnrrerBrc Cs|dkr d}|dkr|}t|t||}ttt|||}||}t|||}|sjt dt |t r|gt|}n t|||}||fS)NrzThere aren't enough frames in the current video to get a clip for the given clip length and frames between clips. The video (and potentially others) will be skipped.) r.r:r=_resample_video_idxr9mathfloorr%r"warningswarn isinstanceslice) rZrnrrerBZ total_framesZ_idxsrdidxsrrrcompute_clips_for_videos    z"VideoClips.compute_clips_for_video)rnrrBrc Cs||_||_||_g|_g|_t|j|jD]6\}}||||||\}}|j ||j |q,t dd|jD}| d |_dS)a Compute all consecutive sequences of clips from video_pts. Always returns clips of size `num_frames`, meaning that the last few frames in a video can potentially be dropped. Args: num_frames (int): number of frames for the clip step (int): distance between two clips frame_rate (int, optional): The frame rate cSsg|] }t|qSrr.rVvrrrrYsz,VideoClips.compute_clips..rN)rnrrBrdresampling_idxsr`rZr[rxappendr#rTZcumsumtolistcumulative_sizes) r*rnrrBrZrerdrwZ clip_lengthsrrrrQs  zVideoClips.compute_clipscCs|Sr() num_clipsr/rrrr0szVideoClips.__len__cCs t|jSr(r-r/rrr num_videosszVideoClips.num_videoscCs |jdS)zJ Number of subclips that are available in the video list. )rr/rrrrszVideoClips.num_clipsr1cCs6t|j|}|dkr|}n||j|d}||fS)zw Converts a flattened representation of the indices into a video_idx, clip_idx representation. rr)bisect bisect_rightr)r*r2 video_idxclip_idxrrrget_clip_location s zVideoClips.get_clip_location)rn original_fpsnew_fpsrcCsPt||}|r(t|}tdd|Stj|tjd|}|tj }|S)NrS) r: is_integerr9rvr#ZarangeZfloat32rrtoint64)rnrrrrwrrrrps  zVideoClips._resample_video_idxc Cst||kr&td|d|d||\}}|j|}|j||}ddlm}|}|dkr|jdkrxtd|j dkrtd|j dkrtd |j dkrtd |j dkrtd |dkr|d }|d  } t||| \} } } nt|} | j}d }tt|d }tt|d  }d\}}tdd}t| jj| jj}| jrt| jj| jj}t|||tj}t|||tj}| j}t||j|j |j |j ||f||j |j||f|d \} } }d|i} |d k r|| d<|j d k r&|j!||}t"|t#j$r||d}| |} |j | d<t%| |j&ksLt'| j(d|j&|j)dkrh| *dddd} | | | |fS)a7 Gets a subclip from a list of videos. Args: idx (int): index of the subclip. Must be between 0 and num_clips(). Returns: video (Tensor) audio (Tensor) info (Dict) video_idx (int): index of the video in `video_paths` zIndex z out of range (z number of clips)r)get_video_backendZpyavz.pyav backend doesn't support _video_width != 0z/pyav backend doesn't support _video_height != 0z6pyav backend doesn't support _video_min_dimension != 0z6pyav backend doesn't support _video_max_dimension != 0z0pyav backend doesn't support _audio_samples != 0rN)rrr) Z video_widthZ video_heightZvideo_min_dimensionZvideo_max_dimensionZvideo_pts_rangevideo_timebaseZ audio_samplesZaudio_channelsZaudio_pts_rangeaudio_timebaser[ audio_fpsz x rL)+r IndexErrorrr'rdZ torchvisionrrEr!rFrGrHrIitemrr r[rr9rr numerator denominatorZ has_audiorrrqrrceilZaudio_sample_raterrJrBr|rur#Tensorr.rnrgshaperKZpermute)r*r2rrZ video_pathZclip_ptsrbackendZ start_ptsZend_ptsZvideoZaudioinfo_infor[rZvideo_start_ptsZ video_end_ptsZaudio_start_ptsZ audio_end_ptsrr_Zresampling_idxrrrget_clip!sz                & zVideoClips.get_clipcCsndd|jD}dd|jD}|r6t|}|}|j}||d<||d<|d=|d=|d=d |d <|S) NcSsg|] }t|qSrryrzrrrrYzsz+VideoClips.__getstate__..cSsg|]}|tjqSr)rr#r)rVr;rrrrYsvideo_pts_sizesrZrdr|rr_version)rZr#catnumpy__dict__copy)r*rrZdrrr __getstate__ys  zVideoClips.__getstate__)rrcCsfd|kr||_dStj|dtjd}tj||ddd}|d=||d<||_||j|j|jdS)NrrZrSrr)r ) rr#rTrsplitrQrnrrB)r*rrZrrr __setstate__szVideoClips.__setstate__) r>rNNrrrrrrrr?)N)N) r4r5r6r7rr8r9r rrr+rOrPpropertyrfro staticmethodr#rr r rvrxrQr0rrrrprrrrrrrr=Isl %  (Xr=)r) rrqrs fractionsrtypingrrrrrr r r r r#Ztorchvision.ior rrrr\rrrrr9rrr%r&r<r=rrrrs ,