U xʹj@s ddlZddlZddlZddlZddlZddlZddlZddlZddlZddl m Z ddl m Z ddl mZmZddlmZddlmZddlmZmZmZdd lmZmZdd lmZmZmZed Z dgZ!dgZ"da#ej$Z$d d Z%ddZ&dDddZ'ddddZ(dEddZ)e ddZ*ddZ+erd>e;ZdS)GN)contextmanager) has_magic) TYPE_CHECKINGIterable)DEFAULT_CALLBACK)FSTimeoutError)LocalFileSystemmake_path_posix trailing_sep)AbstractBufferedFileAbstractFileSystem)glob_translate is_exception other_pathsz_[^_]cCsts tatS)zAllocate or return a threading lock. The lock is allocated on first use to allow setting one lock per forked process. )_lock threadingLockrrJ/var/www/html/bwave/python/venv/lib/python3.8/site-packages/fsspec/asyn.pyget_locksrcCsdtd<dtd<dadS)zReset the global lock. This should be called only on the init of a forked process to reset the lock to None, enabling the new forked process to get a new lock. Nr)iothreadlooprrrrr reset_lock&src st|r|nd}|dk r"tj||d}zBz|IdH|d<Wn*tk r`}z ||d<W5d}~XYnXW5|XdS)Ntimeoutr)asynciowait_forset Exception)eventcororesultrexrrr_runner3s r$rc Os|r|nd}|dks|r$tdztj}||kr@tdWn(tk rXYntk rjYnX|||}dg}t}tt |||||| drq|dk r|d8}|dkrt q|d} t | tj rt | nt | tr| n| SdS)z Make loop run coroutine until it returns. Runs in other thread Examples -------- >>> fsspec.asyn.sync(fsspec.asyn.get_loop(), func, *args, timeout=timeout, **kwargs) NzLoop is not runningz)Calling sync() from within a running looprr)Z is_closed RuntimeErrorreventsget_running_loopNotImplementedErrorrEventZrun_coroutine_threadsafer$waitr isinstance TimeoutError BaseException) rfuncrargskwargsZloop0r!r"r Z return_resultrrrsync?s6       r1cstfdd}|S)zGiven a function, make so can be called in blocking contexts Leave obj=None if defining within a class. Pass the instance if attaching as an attribute of the instance. cs p |d}t|jf||S)Nr)r1r)r/r0selfr.objrrwrapperss zsync_wrapper..wrapper) functoolswraps)r.r4r5rr3r sync_wrapperlsr8c csFt}z,tjdkr,ttdr,ttdVW5t|XdS)NntWindowsSelectorEventLoopPolicy)rZget_event_loop_policyZset_event_loop_policyosnamehasattrr:)Zoriginal_policyrrr_selector_policy{s  r>c CsztddkrrtZtddkrhtttd<W5QRXtjtdjdd}d|_| |t d<W5QRXtdS)zdCreate or return the default fsspec IO loop The loop will be running on a separate thread. rNZfsspecIO)targetr<T) rrr>rZnew_event_looprThreadZ run_foreverdaemonstartr)thrrrget_loops  rDerroriFc Csddlm}|r"d|kr2|dSnd|kr2|dS|r:tStdkrFtSzttj\}}Wnttt fk rxtYSX|tj krdS|dSdS)Nr)confZnofiles_gather_batch_sizeZgather_batch_size) Z fsspec.configrG_NOFILES_DEFAULT_BATCH_SIZEresource_DEFAULT_BATCH_SIZEZ getrlimitZ RLIMIT_NOFILE ImportError ValueError ResourceErrorZ RLIM_INFINITY)nofilesrGZ soft_limit_rrr_get_batch_sizes"    rR)returncCs*ztWdStk r$YdSXdS)z Being executed by an event loop?TFN)rr'r%rrrr running_asyncs rTcs|dkrt|d}|dkr"t|}|dks.tfdd}d}t|}dg|} t} | sf||krt| |kr||kr| t|||||d7}qf| sqtj| tjdIdH\} } | rZ| IdH\} } | | | <qqZ| S) aSRun the given coroutines in chunks. Parameters ---------- coros: list of coroutines to run batch_size: int or None Number of coroutines to submit/wait on simultaneously. If -1, then it will not be any throttling. If None, it will be inferred from _get_batch_size() callback: fsspec.callbacks.Callback instance Gets a relative_update when each coroutine completes timeout: number or None If given, each coroutine times out after this time. Note that, since there are multiple batches, the total run time of this function will in general be longer return_exceptions: bool Same meaning as in asyncio.gather nofiles: bool If inferring the batch_size, does this operation involve local files? If yes, you normally expect smaller batches. N)rPrHrc snz\ztj|dIdH|fWW>Stk rX}zs8||fWYWSd}~XYnXW5dXdS)Nrr)Zrelative_updaterrr)r!iecallbackreturn_exceptionsrrr _run_coros&z'_run_coros_in_chunks.._run_coror)Z return_when) rRlenAssertionErrorraddrZ ensure_futurer*ZFIRST_COMPLETEDpop)coros batch_sizerXrrYrPrZrUnresultspendingdoner"krrWr_run_coros_in_chunkss*       rf_ls _cat_file _get_file _put_file_rm_file_cp_file _pipe_file _expand_path_info_isfile_isdir_exists_walk_glob_find_du_size_mkdir _makedirscsheZdZdZdZdZdZddddfdd Zedd Z d d Z dKd d Z ddZ ddZ dLddZdMddZdNddZddZdOddZdPddZdQd!d"ZdRd#d$Zdeddfd%d&Zd'd(Zdedfd)d*Zd+d,Zd-d.Zd/d0ZdSd1d2Zd3d4Zd5d6ZdTd7d8Z dUd:d;Z!dVdd?Z#dXd@dAZ$dYdBdCZ%dZdDdEZ&d[dFdGZ'd\dIdJZ(Z)S)]AsyncFileSystemakAsync file operations, default implementations Passes bulk operations to asyncio.gather for concurrent operation. Implementations that have concurrent batch operations and/or async methods should inherit from this class instead of AbstractFileSystem. Docstrings are copied from the un-underscored method in AbstractFileSystem, if not given. TFN) asynchronousrr`cs@||_t|_|s"|pt|_nd|_||_tj||dSN) r{r;getpid_pidrD_loopr`super__init__)r2r{rr`r/r0 __class__rrr;s zAsyncFileSystem.__init__cCs|jtkrtd|jS)NzThis class is not fork-safe)r~r;r}r%rr2rrrrEszAsyncFileSystem.loopcstdSr|r(r2pathr0rrrrkKszAsyncFileSystem._rm_filecsF|pj}j||dIdH}tfddt|D|ddIdHS)N recursivecsg|]}j|fqSr)rk.0pr0r2rr Ssz'AsyncFileSystem._rm..Tr`rP)r`rnrfreversed)r2rrr`r0rrr_rmNs zAsyncFileSystem._rmcstdSr|r)r2path1path2r0rrrrlXszAsyncFileSystem._cp_filecs&|||IdH||IdHdSr|)rlrk)r2rrrrr_mv_file[szAsyncFileSystem._mv_filecsb|dkr|rd}n |dkrd}t|tr.rexistsflattencs g|]\}}j||fqSr)rl)rp1Zp2rrrrsT)r`rYrP)r+liststrrnr[r rqrrr`ziprffilterrFileNotFoundError)r2rrron_errorrr`r0Zpaths1Zpaths2 source_is_strsource_is_file dest_is_dirrr_r"r#rrr_copy_sZ          zAsyncFileSystem._copy overwritecstdSr|r)r2rvaluemoder0rrrrmszAsyncFileSystem._pipe_filecsDt|tr||i}|pj}tfdd|D|ddIdHS)Ncs g|]\}}j||fqSr)rmrrevrrrrsz)AsyncFileSystem._pipe..Tr)r+rr`rfitems)r2rrr`r0rrr_pipes  zAsyncFileSystem._pipecsd}d}|dk rR|dkrR|dkr0| }d}d}q^|pF||IdHd}||}n |dkr^d}|s|dk r|dkr|dk r|p||IdHd}||}n |dkrd}t|tjr|d8}d|d |S) z"Helper for "Range"-based _cat_fileNFrTsizerzbytes=-)ror+numbersIntegral)r2urlrBendrsuffrrr_process_limitss*   zAsyncFileSystem._process_limitscstdSr|r)r2rrBrr0rrrrhszAsyncFileSystem._cat_filerc sj||dIdH}fdd|D}|p0j}t||dddIdH}dkrhttt|d} | rh| t|dkst|ts|d  |krfd d t ||DS|d SdS) Nrcsg|]}j|fqSrrh)rrrrrrsz(AsyncFileSystem._cat..Tr`rPrYrFrrcs&i|]\}}dkst|s||qS)omit)rr)rrr s z(AsyncFileSystem._cat..) rnr`rfnextrrr[r+r_strip_protocolr) r2rrrr`r0pathsr_outr#r)r0rr2r_cats.    zAsyncFileSystem._catrSc s|dk r tt|tstt|ts2|gt|}t|tsJ|gt|}t|t|ksjt|t|krntfddt|||D}|pj}t ||dddIdHS)aYGet the contents of byte ranges from one or more files Parameters ---------- paths: list A list of of filepaths on this filesystems starts, ends: int or list Bytes limits of the read. If using a single int, the same value will be used to read all the specified files. Ncs*g|]"\}}}j|f||dqS))rBrr)rrsrVrrrrsz/AsyncFileSystem._cat_ranges..Tr) r(r+r TypeErrorrr[rNrr`rf) r2rZstartsZendsZmax_gapr`rr0r_rrr _cat_rangess(       zAsyncFileSystem._cat_rangescstdSr|r)r2lpathrpathrr0rrrrjszAsyncFileSystem._put_filecst|trt|tr|}|} nt|t} | r4t|}tj|||d} | rt|rZ|dk rtfdd| D} | stdSt| dk} t|tot|p|IdH}  |}| ot |r| pt | o| ot| } t | || | d}dd| Dfd dt | |D}fd dt | |D}t jfd d|DIdH|pLj}g}|t||D]*\}}|j}||||f|qdt|||d IdHS) aCopy file(s) from local. Copies a specific file or tree of files (if recursive=True). If rpath ends with a "/", it will be assumed to be a directory, and target files will go within. The put_file method will be called concurrently on a batch of files. The batch_size option can configure the amount of futures that can be executed at the same time. If it is -1, then all the files will be uploaded concurrently. The default can be set for this instance by passing "batch_size" in the constructor, or for all instances by setting the "gather_batch_size" key in ``fsspec.config.conf``, falling back to 1/8th of the system limit . rrNcs"g|]}t|s|s|qSr)r isdirr)fsrrr/s z(AsyncFileSystem._put..rrcSsi|]}|tj|qSr)r;rr)rlrrrrDsz(AsyncFileSystem._put..csg|]\}}|r|qSrrrrris_dirrrrEscs g|]\}}|s||fqSrrrrrrrFscsg|]}j|ddqST)exist_ok)ry)rdrrrrHsr`rX)r+rrr r Z expand_pathr[r rqrrrrrZgatherr`set_size branch_cororjappendrf)r2rrrrXr`rr0rpathslpathsrrrrZrdirsZ file_pairsr_ZlfilerfileZput_filer)rrr2r_put sT        zAsyncFileSystem._putcstdSr|r)r2rrr0rrrriUszAsyncFileSystem._get_filecsdt|trt|tr|}|}nt|t} | o4t| } |}j|||dIdH}| r|rf|dk rfdd|DIdH}|sdSt|}t|dk} t|tot|pt |} | ot |r| pt | o| o| } t ||| | d}dd|D| dj }g}|t|t||D]*\}}|j}||||f|q$t|||d IdHS) aCopy file(s) to local. Copies a specific file or tree of files (if recursive=True). If lpath ends with a "/", it will be assumed to be a directory, and target files will go within. Can submit a list of paths, which may be glob-patterns and will be expanded. The get_file method will be called concurrently on a batch of files. The batch_size option can configure the amount of futures that can be executed at the same time. If it is -1, then all the files will be uploaded concurrently. The default can be set for this instance by passing "batch_size" in the constructor, or for all instances by setting the "gather_batch_size" key in ``fsspec.config.conf``, falling back to 1/8th of the system limit . rNcs(g|] }t|s|IdHs|qSr|rrrrrr~sz(AsyncFileSystem._get..rrcSs"g|]}tjtj|ddqSr)r;makedirsrdirname)rZlprrrrsr`r)r+rrr rrnr r[r rrrr^r`rrrrirrf)r2rrrrXrr0rrrZsource_not_trailing_seprrrr`r_Zget_filerrr_getXsX         zAsyncFileSystem._getcs.z||IdHddkWSYdSXdS)NtypefileF)ror2rrrrrpszAsyncFileSystem._isfilecs6z||IdHddkWStk r0YdSXdS)Nr directoryF)roOSErrorrrrrrqszAsyncFileSystem._isdircs||IdHddS)Nr)rogetrrrrrwszAsyncFileSystem._sizecs*|pj}tfdd|D|dIdHS)Ncsg|]}|qSr)rwrrrrrsz*AsyncFileSystem._sizes..)r`)r`rf)r2rr`rrr_sizess  zAsyncFileSystem._sizescs6z|j|f|IdHWdStk r0YdSXdS)NTF)rorrrrrrrs zAsyncFileSystem._existscstdSr|rrrrrroszAsyncFileSystem._infocstdSr|r)r2rdetailr0rrrrgszAsyncFileSystem._lsrc Ks|dk r|dkrtd||}i}i}i}|dd}z |j|fddi|IdH} Wnfttfk r} zD|dkr|nt|r|| |r|iifVn |ggfVWYdSd} ~ XYnX| D]h} | dd} | ddd } | d d kr| |kr| || <| || <q| |kr&| |d <q| || <q|rD|||fVn|t |t |fV|dk rx|d8}|dkrxdS|D]:}|j ||f||d |2z3dHW}|Vq6q|dS)Nrmaxdepth must be at least 1rFTrr</rHrrr)rr) rNrr^rgrrcallablerstriprsplitrrs)r2rrrr0Z full_dirsdirsfilesrZlistingrVinfopathnamer<rrQrrrrssX          zAsyncFileSystem._walkcsf|dk r|dkrtdddl}tjjr8tjjtjjfntjjf}||}||}|pp|tdd|D| ddkr| dnt |}| ddkr| dnt |}| ddkr| dnt |} t ||| } | d d } t |s@|j|f|IdHr0| s|gS||j|f|IdHiSn| s:gSiSnnd |d| kr|d| d } |d| d} || ddd d} nd } || ddd d} d |kr|dk r| d }||dd d}| ||} nd} |j| f| ddd|IdH}t||r"d nd |fddt|D}| rZ|St|SdS)Nrrrcss|]}|dVqdS)**Nr)rseprrr sz(AsyncFileSystem._glob..*?[rFrrrT)rwithdirsrcs8i|]0\}}r(|ddkr(|dn|r||qS)rrr)match)rrrZappend_slash_to_dirnamepatternrrr/s z)AsyncFileSystem._glob..)rNrer;raltseprendswithrtuplefindr[minr^rrrrorindexcountrurcompilesortedrr)r2rrr0rsepsZ ends_with_sepZidx_starZ idx_qmarkZ idx_bracemin_idxrrootdepthZidx_double_starsZdepth_double_starsZallpathsrrrrrtsf"                zAsyncFileSystem._globcs^i}|j|fd|i|IdHD]$}||IdH}|d||d<q |rVt|S|SdS)Nrrr<)rurosumvalues)r2rtotalrr0sizesfrrrrrv>s  zAsyncFileSystem._duc s||}i|dd}|rJ|dkrJ||IdHrJ||IdH|<|j||fddi|2z83dHW\}}}|r||dd|Dqb6s||IdHri|<t} |s| Sfdd| DSdS)NrFrTcSsi|]\}}|d|qS)r<r)rr<rrrrrWsz)AsyncFileSystem._find..csi|]}||qSrr)rr<rrrr`s) rr^rqrorsupdaterrpr) r2rrrr0rrQrrnamesrr rruIs  * zAsyncFileSystem._findcs8|dk r|dkrtdt|tr:|g||IdH}nt}fdd|D}|D]}t|rtj||dIdH}||O}|rV|dk r|dkrqV|tjt|||dk r|dnddIdHO}qVn&|rtj||ddIdH}||O}||krV|d ks |IdHrV| |qV|s0t |t |S) Nrrcsg|]}|qSr)rrrrrrjsz0AsyncFileSystem._expand_path..)rrT)rrF) rNr+rrnrrrtrrurrr]rr)r2rrrrrbitZrecrrrrnbs:  " zAsyncFileSystem._expand_pathcsdSr|r)r2rZcreate_parentsr0rrrrxszAsyncFileSystem._mkdircsdSr|r)r2rrrrrryszAsyncFileSystem._makedirsrbcsd|ks|drttdS)Nb compression)rrNr()r2rrr0rrr open_asyncszAsyncFileSystem.open_async)FN)FNNN)r)NN)NN)FrN)NNrS)r)N)T)Nr)N)TN)NF)FN)T)F)r )*__name__ __module__ __qualname____doc__Z async_implmirror_sync_methodsZdisable_throttlingrpropertyrrkrrlrrrmrrrhrrrjrrrirrprqrwrrrrorgrsrtrvrurnrxryr __classcell__rrrrrz*sl     <     )  J I   6 E  %  rzcCsddlm}tttD]}|ds(q|dd}t|rt t ||d}t t ||ddd}|t ||dk}|r|rt t |||d }t ||||j st t ||dd d|_ qdS) a,Populate sync and async methods for obj For each method will create a sync version if the name refers to an async method (coroutine) and there is no override in the child class; will create an async method for the corresponding sync method if there is no implementation. Uses the methods specified in - async_methods: the set that an implementation is expected to provide - default_async_methods: that can be derived from their sync version in AbstractFileSystem - AsyncFileSystem: async-specific default coroutines r)r rQrNF__func__r)r4r)Zfsspecr async_methodsdirrz startswithprivaterinspectiscoroutinefunctiongetattrr8setattrr)r4r methodZsmethodZiscoZunsync is_defaultZmthrrrrs$      rc@s eZdZdS)FSSpecCoroutineCancelN)rrrrrrrr#sr#Tc sddlddtjtdD}|r4dd|Dfdd|D}|r|D]hj}tjj |tjjfdd|Dzj |WqP|k rYqPXqP|S)NrcSsg|]}|s|qSr)rd)rtrrrrsz'_dump_running_tasks..cSsg|] }|qSr) print_stackrtaskrrrrsc sLg|]D}|jjj|jjjj|jjjj|jjj|jjr@|nddqS)N)localsr firstlineZlinelostackr')_corocr_framef_localsf_code co_filenameco_firstlinenof_lineno format_stackr&) traceback with_taskrrrs     csg|]}|dqS)rr)rcb)r$rrrs) r3rtasksZ all_tasksrZ _callbackscancelZfuturesFutureZ set_exceptionr+throw)Zprintoutr7excr4r6rZcbsr)r$r3r4r_dump_running_taskss&  r;c@sZeZdZdddZddZddZdd d Zd d Zd dZddZ ddZ dddZ dS)AbstractAsyncStreamedFilerHcs|dkr dnt|}|jdkr&td|dkr:|j|j}|jrHtd|dkrTdS||j|j|IdH}|jt|7_|S)z Return data from cache, or fetch pieces as necessary Parameters ---------- length: int (-1) Number of bytes to read; if <0, all remaining bytes. NrHr zFile not in read moderI/O operation on closed file.)intrrNrlocclosed _fetch_ranger[)r2lengthrrrrreads   zAbstractAsyncStreamedFile.readcsj|jdkrtd|jr td|jr.td|j|}|j|7_|j|jkrf| IdH|S)z Write data to buffer. Buffer only sent on flush() or if buffer is greater than or equal to blocksize. Parameters ---------- data: bytes Set of bytes to be written. >wbabzFile not in write moder=z0This file has been force-flushed, can only closeN) rrNrAforcedbufferwriter@tell blocksizeflush)r2datarrrrrIs  zAbstractAsyncStreamedFile.writecs|t|ddrdS|jrdS|jdkr,d|_nF|jsD|jddIdH|jdk rr|j|j|j|j |jd|_dS)z=Close file Finalizes writes, discards cache Z _unclosableFNr T)force) rrArcacherGrLrZinvalidate_cacherZ_parentrrrrclose s   zAbstractAsyncStreamedFile.closeFcs|jrtd|r |jr td|r*d|_|jdkr8dS|sP|j|jkrPdS|jdkrd|_z|IdHWnd|_YnX|j |dIdHdk r|j|j dd7_t |_dS) NzFlush on closed filez+Force flush cannot be called more than onceT>rErFr)finalF) rArNrGrrHrJrKoffset_initiate_upload _upload_chunkseekioBytesIO)r2rNrrrrL s(   zAbstractAsyncStreamedFile.flushcs|Sr|rrrrr __aenter__=sz$AbstractAsyncStreamedFile.__aenter__cs|IdHdSr|)rP)r2exc_typeexc_valexc_tbrrr __aexit__@sz#AbstractAsyncStreamedFile.__aexit__cstdSr|r)r2rBrrrrrBCsz&AbstractAsyncStreamedFile._fetch_rangecsdSr|rrrrrrTFsz*AbstractAsyncStreamedFile._initiate_uploadcstdSr|r)r2rQrrrrUIsz'AbstractAsyncStreamedFile._upload_chunkN)rH)F)F) rrrrDrIrPrLrYr]rBrTrUrrrrr<s  r<)N)N)F)?rZasyncio.eventsr6rrWrr;rr contextlibrglobrtypingrr callbacksr exceptionsrZimplementations.localr r r specr r utilsrrrrrrrrr'rrr$r1r8r>rDrKrErOrMrrrLrJrRboolrTrfrrzrrr#r;r<rrrrs       -        Gm  !