U ʹjGd @sUdZddlZddlZddlZddlZddlZddlZddlmZddl m Z ddl m Z m Z mZmZmZmZmZddlZddlmZmZddlmZddlmZdd lmZd d lmZd d lmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%d dlm&Z&d dl'm(Z(d dl)m*Z*d dl+m,Z,e&-e.Z/dZ0dZ1dZ2ej3dej4dZ5GdddeZ6GdddeZ7ej8dddZ9e gej8fZ:e9a;e:eddd$d%Z?ee@e@ej8d&d'd(ZAd)d d*ejBejCejDjEfd+d,e,eFe@eGeGeeeHeeeHd-ffee@ee@d-ffed.d/d0ZIeFeeFeFd1d2d3ZJdAeeeFdd4d5d6ZKee#eFee#d7d8d9ZLejeFd:d;d<ZMe3d=ejNZOeeFe@eeFd>d?d@ZPdS)Bz>Contains utilities to handle HTTP requests in Huggingface Hub.N) lru_cache)quote)AnyCallableListOptionalTupleTypeUnion) HTTPErrorResponse) HTTPAdapter)PreparedRequest)OfflineModeIsEnabled) constants)BadRequestErrorDisabledRepoErrorEntryNotFoundErrorGatedRepoErrorHfHubHTTPErrorRepositoryNotFoundErrorRevisionNotFoundError)logging)JSONDecodeError) SliceFileObj) HTTP_METHOD_TX-Amzn-Trace-Idz x-request-idz x-amz-cf-ida  # staging or production endpoint ^https://[^/]+ ( # on /api/repo_type/repo_id /api/(models|datasets|spaces)/(.+) | # or /repo_id/resolve/revision/... /(.+)/resolve/(.+) ) )flagscs4eZdZdZfddZeedfdd ZZS)UniqueRequestIdAdapterrc stj|f|t|jkr8|jtp0tt|jt<t t|jdddk}t d|jtd|j d|j d|d dS) N authorizationrzRequest :  z (authenticated: ))super add_headersX_AMZN_TRACE_IDheadersget X_REQUEST_IDstruuiduuid4lenloggerdebugmethodurl)selfrequestkwargsZ has_token __class__Z/var/www/html/bwave/python/venv/lib/python3.8/site-packages/huggingface_hub/utils/_http.pyr'Ns &z"UniqueRequestIdAdapter.add_headersr5returnc stjrtdt|ztj|f||WStjk r}z0|j t }|dk rn|j d|df|_ W5d}~XYnXdS)zSCatch any RequestException to append request id to the error message for debugging.zSend: Nz (Request ID: r%) rZHF_DEBUGr0r1_curlifyr&sendrequestsRequestExceptionr)r*r(args)r4r5rAr6e request_idr7r9r:r>[s zUniqueRequestIdAdapter.send) __name__ __module__ __qualname__r(r'rr r> __classcell__r9r9r7r:r Ks r c@seZdZeedddZdS)OfflineAdapterr;cOstd|jddS)Nz Cannot reach za: offline mode is enabled. To disable it, please unset the `HF_HUB_OFFLINE` environment variable.)rr3)r4r5rAr6r9r9r:r>js zOfflineAdapter.sendN)rDrErFrr r>r9r9r9r:rHisrH)r<cCsLt}tjr,|dt|dtn|dt|dt|S)Nzhttp://zhttps://)r?SessionrZHF_HUB_OFFLINEmountrHr )sessionr9r9r:_default_backend_factorypsrL_GLOBAL_BACKEND_FACTORY)backend_factoryr<cCs|atdS)a Configure the HTTP backend by providing a `backend_factory`. Any HTTP calls made by `huggingface_hub` will use a Session object instantiated by this factory. This can be useful if you are running your scripts in a specific environment requiring custom configuration (e.g. custom proxy or certifications). Use [`get_session`] to get a configured Session. Since `requests.Session` is not guaranteed to be thread-safe, `huggingface_hub` creates 1 Session instance per thread. They are all instantiated using the same `backend_factory` set in [`configure_http_backend`]. A LRU cache is used to cache the created sessions (and connections) between calls. Max size is 128 to avoid memory leaks if thousands of threads are spawned. See [this issue](https://github.com/psf/requests/issues/2766) to know more about thread-safety in `requests`. Example: ```py import requests from huggingface_hub import configure_http_backend, get_session # Create a factory function that returns a Session with configured proxies def backend_factory() -> requests.Session: session = requests.Session() session.proxies = {"http": "http://10.10.1.10:3128", "https": "https://10.10.1.11:1080"} return session # Set it as the default session factory configure_http_backend(backend_factory=backend_factory) # In practice, this is mostly done internally in `huggingface_hub` session = get_session() ``` N)rMreset_sessions)rNr9r9r:configure_http_backends rPcCstttdS)a Get a `requests.Session` object, using the session factory from the user. Use [`get_session`] to get a configured Session. Since `requests.Session` is not guaranteed to be thread-safe, `huggingface_hub` creates 1 Session instance per thread. They are all instantiated using the same `backend_factory` set in [`configure_http_backend`]. A LRU cache is used to cache the created sessions (and connections) between calls. Max size is 128 to avoid memory leaks if thousands of threads are spawned. See [this issue](https://github.com/psf/requests/issues/2766) to know more about thread-safety in `requests`. Example: ```py import requests from huggingface_hub import configure_http_backend, get_session # Create a factory function that returns a Session with configured proxies def backend_factory() -> requests.Session: session = requests.Session() session.proxies = {"http": "http://10.10.1.10:3128", "https": "https://10.10.1.11:1080"} return session # Set it as the default session factory configure_http_backend(backend_factory=backend_factory) # In practice, this is mostly done internally in `huggingface_hub` session = get_session() ```  process_id thread_id)_get_session_from_cacheosgetpid threading get_identr9r9r9r: get_sessionsrYcCs tdS)zReset the cache of sessions. Mostly used internally when sessions are reconfigured or an SSLError is raised. See [`configure_http_backend`] for more details. N)rT cache_clearr9r9r9r:rOsrO)rRrSr<cCstS)z Create a new session per thread using global factory. Using LRU cache (maxsize 128) to avoid memory leaks when using thousands of threads. Cache is cleared when `configure_http_backend` is called. )rMrQr9r9r:rTsrT)iiii) max_retriesbase_wait_time max_wait_timeretry_on_exceptionsretry_on_status_codes.)r2r3r]r^r_r`rar<c Ksvt|tr|f}t|tr |f}d}|} d} d|krTt|dtjtfrT|d} t} |d7}zt| dk rz|d| | j f||d|} | j |kr| WSt d| j d|d|||kr| | WSWnb|k r8} zBt d | d |d|t| tjrt||kr(| W5d} ~ XYnXt d | d |d |dt| t|| d} qZdS)a7 Wrapper around requests to retry calls on an endpoint, with exponential backoff. Endpoint call is retried on exceptions (ex: connection timeout, proxy error,...) and/or on specific status codes (ex: service unavailable). If the call failed more than `max_retries`, the exception is thrown or `raise_for_status` is called on the response object. Re-implement mechanisms from the `backoff` library to avoid adding an external dependencies to `hugging_face_hub`. See https://github.com/litl/backoff. Args: method (`Literal["GET", "OPTIONS", "HEAD", "POST", "PUT", "PATCH", "DELETE"]`): HTTP method to perform. url (`str`): The URL of the resource to fetch. max_retries (`int`, *optional*, defaults to `5`): Maximum number of retries, defaults to 5 (no retries). base_wait_time (`float`, *optional*, defaults to `1`): Duration (in seconds) to wait before retrying the first time. Wait time between retries then grows exponentially, capped by `max_wait_time`. max_wait_time (`float`, *optional*, defaults to `8`): Maximum duration (in seconds) to wait before retrying. retry_on_exceptions (`Type[Exception]` or `Tuple[Type[Exception]]`, *optional*): Define which exceptions must be caught to retry the request. Can be a single type or a tuple of types. By default, retry on `requests.Timeout`, `requests.ConnectionError` and `requests.exceptions.ChunkedEncodingError`. retry_on_status_codes (`int` or `Tuple[int]`, *optional*, defaults to `(500, 502, 503, 504)`): Define on which status codes the request must be retried. By default, 5xx errors are retried. **kwargs (`dict`, *optional*): kwargs to pass to `requests.request`. Example: ``` >>> from huggingface_hub.utils import http_backoff # Same usage as "requests.request". >>> response = http_backoff("GET", "https://www.google.com") >>> response.raise_for_status() # If you expect a Gateway Timeout from time to time >>> http_backoff("PUT", upload_url, data=data, retry_on_status_codes=504) >>> response.raise_for_status() ``` > [!WARNING] > When using `requests` it is possible to stream data by passing an iterator to the > `data` argument. On http backoff this is a problem as the iterator is not reset > after a failed call. This issue is mitigated for file objects or any IO streams > by saving the initial position of the cursor (with `data.tell()`) and resetting the > cursor between each call (with `data.seek()`). For arbitrary iterators, http backoff > will fail. If this is a hard constraint for you, please let us know by opening an > issue on [Github](https://github.com/huggingface/huggingface_hub). rNdatar)r2r3z HTTP Error z thrown while requesting r$'z' thrown while requesting z Retrying in z s [Retry /z].r) isinstancetypeintioIOBasertellrYseekr5 status_coder0warningraise_for_statusr?ConnectionErrorrOtimesleepmin)r2r3r]r^r_r`rar6Znb_triesZ sleep_timeZio_obj_initial_posrKresponseerrr9r9r: http_backoffs<D       ru)r3endpointr<cCsD|r|dntj}|tjtjfkr@|tj|}|tj|}|S)zReplace the default endpoint in a URL by a custom one. This is useful when using a proxy and the Hugging Face Hub returns a URL with the default endpoint. rd)rstriprZENDPOINTZ_HF_DEFAULT_ENDPOINTZ_HF_DEFAULT_STAGING_ENDPOINTreplace)r3rvr9r9r:fix_hf_endpoint_in_urlPs ry)rs endpoint_namer<c Csz|Wnxtk r}zV|jd}|jd}|dkrt|jddd|jd}tt|||n|dkr|jddd |jd}tt|||n|d kr|jddd |jd}tt |||n|d kr&|jddd |jddd }tt |||n>|dkst|jdkr|dkr|j dk r|j jdk rt |j jdk r|jddd|jdd}tt|||n|jdkr|dk rd|dnd}tt|||n|jdkrd|jd|dd|jdd}tt|||nF|jdkrd|j jd}|d|d |jd!d}tt|||ttt|||W5d}~XYnXdS)"a( Internal version of `response.raise_for_status()` that will refine a potential HTTPError. Raised exception will be an instance of `HfHubHTTPError`. This helper is meant to be the unique method to raise_for_status when making a call to the Hugging Face Hub. Example: ```py import requests from huggingface_hub.utils import get_session, hf_raise_for_status, HfHubHTTPError response = get_session().post(...) try: hf_raise_for_status(response) except HfHubHTTPError as e: print(str(e)) # formatted message e.request_id, e.server_message # details returned by server # Complete the error message with additional information once it's raised e.append_to_message(" `create_commit` expects the repository to exist.") raise ``` Args: response (`Response`): Response from the server. endpoint_name (`str`, *optional*): Name of the endpoint that has been called. If provided, the error message will be more complete. > [!WARNING] > Raises when the request has failed: > > - [`~utils.RepositoryNotFoundError`] > If the repository to download from cannot be found. This may be because it > doesn't exist, because `repo_type` is not set correctly, or because the repo > is `private` and you do not have access. > - [`~utils.GatedRepoError`] > If the repository exists but is gated and the user is not on the authorized > list. > - [`~utils.RevisionNotFoundError`] > If the repository exists but the revision couldn't be find. > - [`~utils.EntryNotFoundError`] > If the repository exists but the entry (e.g. the requested file) couldn't be > find. > - [`~utils.BadRequestError`] > If request failed with a HTTP 400 BadRequest error. > - [`~utils.HfHubHTTPError`] > If request failed for a reason not listed above. z X-Error-CodeX-Error-MessageZRevisionNotFoundz Client Error. zRevision Not Found for url: .Z EntryNotFoundzEntry Not Found for url: Z GatedRepoz!Cannot access gated repo for url z$Access to this resource is disabled.z!Cannot access repository for url  Z RepoNotFoundiz+Invalid credentials in Authorization headerNzRepository Not Found for url: z Please make sure you specified the correct `repo_id` and `repo_type`. If you are trying to access a private or gated repo, make sure you are authenticated. For more details, see https://huggingface.co/docs/huggingface_hub/authenticationiz Bad request for z endpoint:z Bad request:iz Forbidden: z Cannot access content at: z2 Make sure your token has the correct permissions.iRangez. Requested range: z. Content-Range: z Content-Range)rnr r)r*rlr3_formatrrrrr5REPO_API_REGEXsearchrrrr,)rsrzrBZ error_code error_messagemessageZ range_headerr9r9r:hf_raise_for_status]s5              r) error_typecustom_messagersr<cCsg}|jd}|dk r"||zp|}|d}|dk r^t|trT||n |||d}|dk r|D]}d|krt||dqtWn@tk r|jdd}|jrd| kr||jYnXdd |D}tt |}d |} |} | r:| | kr:d |kr.| d | 7} n | d | 7} d} d} t d ftd ftdffD]<\} }|j| }|rXt|} d|d|d} qqX| r| | krd | kr| d }| d|| | |d} n| | 7} || || pddS)Nr{errorerrorsrz Content-Typer"htmlcSs$g|]}t|rt|qSr9)r,strip).0liner9r9r: s z_format..r~r|z Request IDz Amzn Trace IDz Amz CF IDz (r#r%)rsserver_message)r)r*appendjsonrelistextendrtextlowerdictfromkeysjoinr+r( X_AMZ_CF_IDr,indexr)rrrsZ server_errorsZ from_headersrbrr content_typerZfinal_error_messagerCZrequest_id_messageheaderlabelvalueZ newline_indexr9r9r:rs\               rr;cCsdd|jfg}t|jD].\}}|dkr4d}|dd||fg7}q|jr|j}t|trr|j ddd }nt |d rd }t |d kr|d d d}|d| ddfg7}|d |j fg7}g}|D],\}}|r|t||r|t|qd|S)zConvert a `requests.PreparedRequest` into a curl command (str). Used for debug purposes only. Implementation vendored from https://github.com/ofw/curlify/blob/master/curlify.py. MIT License Copyright (c) 2016 Egor. )ZcurlNz-Xr!zz-Hz{0}: {1}zutf-8ignore)rreadziNz ... [truncated]z-dr~r"r$)r2sortedr)itemsrformatbodyrebytesdecodehasattrr/rxr3rrr)r5partskvrZ flat_partsr9r9r:r=0s0      r=z%^\s*bytes\s*=\s*(\d*)\s*-\s*(\d*)\s*$)original_range resume_sizer<cCs|sd|dSd|kr(td|dt|}|sFtd|d|\}}|s|sjtd|dt||}d|}|d krtd |d|St|}||}|rt|}d|d|}||krtd |d|Sd|dS) zB Adjust HTTP Range header to account for resume position. zbytes=-,zMultiple ranges detected - z, not supported yet.zInvalid range format - r}zbytes=-rzEmpty new range - ) ValueError RANGE_REGEXmatch RuntimeErrorgroupsrg)rrrstartendZ new_suffixZ new_range new_startr9r9r:_adjust_range_header\s2     r)N)Q__doc__rhrUrerWrpr- functoolsrshlexrtypingrrrrrr r r?r r Zrequests.adaptersr Zrequests.modelsrZhuggingface_hub.errorsrr"rrrrrrrrrrZ_fixesrZ_lfsrZ_typingrZ get_loggerrDr0r(r+rcompileVERBOSErr rHrIrLZBACKEND_FACTORY_TrM__annotations__rPrYrOrgrTTimeoutro exceptionsChunkedEncodingErrorr,float Exceptionruryrrr= IGNORECASErrr9r9r9r:s~  $    $        $    { Q)