U e37@sUdZddlmZddlZddlZddlmZddlmZddl m Z m Z ddl Z ddl mZmZmZmZe rzddlmZd d lmZd d lmZd Zd ZdZdZdZeedjddZ Gddde!Z"Gddde"Z#Gddde"Z$dddddZ%Gdd d ej&Z'Gd!d"d"Z(d#d$d%d&Z)d#d$d'd(Z*d#d$d)d*Z+d#d$d+d,Z,dZ-d-e.d.<e,re*re)se+se(Z-ndZ-d/d0d1d2d3Z/d4a0d5d$d6d7Z1d4a2d5d$d8d9Z3d/d:d1d;d<Z4d=d$d>d?Z5d#d$d@dAZ6dS)Ba Support for streaming http requests in emscripten. A few caveats - Firstly, you can't do streaming http in the main UI thread, because atomics.wait isn't allowed. Streaming only works if you're running pyodide in a web worker. Secondly, this uses an extra web worker and SharedArrayBuffer to do the asynchronous fetch operation, so it requires that you have crossOriginIsolation enabled, by serving over https (or from localhost) with the two headers below set: Cross-Origin-Opener-Policy: same-origin Cross-Origin-Embedder-Policy: require-corp You can tell if cross origin isolation is successfully enabled by looking at the global crossOriginIsolated variable in javascript console. If it isn't, streaming requests will fallback to XMLHttpRequest, i.e. getting the whole request into a buffer and then returning it. it shows a warning in the javascript console in this case. Finally, the webworker which does the streaming fetch is created on initial import, but will only be started once control is returned to javascript. Call `await wait_for_streaming_ready()` to wait for streaming fetch. NB: in this code, there are a lot of javascript objects. They are named js_* to make it clear what type of object they are. ) annotationsN)Parser)files) TYPE_CHECKINGAny)JsArray JsExceptionJsProxyto_js)Buffer)EmscriptenRequest)EmscriptenResponse)z user-agentzemscripten_fetch_worker.jszutf-8)encodingcs0eZdZd dddddddfddZZS) _RequestErrorNrequestresponsez str | NonezEmscriptenRequest | NoneEmscriptenResponse | None)messagerrcs$||_||_||_t|jdSN)rrrsuper__init__)selfrrr __class__N/opt/hc_python/lib/python3.8/site-packages/urllib3/contrib/emscripten/fetch.pyrCsz_RequestError.__init__)N)__name__ __module__ __qualname__r __classcell__r r rr!rBs rc@s eZdZdS)_StreamingErrorNr"r#r$r r r r!r&Psr&c@s eZdZdS) _TimeoutErrorNr'r r r r!r(Tsr(zdict[str, Any]r )dict_valreturncCst|tjjdS)N)Zdict_converter)r jsZObjectZ fromEntries)r)r r r!_obj_from_dictXsr,cseZdZdddddddddZd d d d Zd d ddZed d ddZd d fdd Zd d ddZ d d ddZ d d ddZ dddddZ Z S) _ReadStreamrfloatr intr ) int_buffer byte_buffertimeoutworker connection_idrcCsT||_||_d|_d|_||_||_|dkr8td|nd|_d|_d|_ ||_ dS)NrTF) r0r1read_posread_lenr4r3r/r2is_live _is_closedr)rr0r1r2r3r4rr r r!r]s z_ReadStream.__init__Noner*cCs |dSr)closerr r r!__del__qsz_ReadStream.__del__boolcCs|jSr)r9r=r r r! is_closedusz_ReadStream.is_closedcCs|Sr)r@r=r r r!closedysz_ReadStream.closedcs\|sXd|_d|_d|_d|_d|_d|_|jrN|j t d|j id|_t dS)NrTr<F)r@r7r6r0r1r9rr8r3 postMessager,r4rr<r=rr r!r<}sz_ReadStream.closecCsdSNTr r=r r r!readablesz_ReadStream.readablecCsdSNFr r=r r r!writablesz_ReadStream.writablecCsdSrEr r=r r r!seekablesz_ReadStream.seekabler )byte_objr*cCs>|jstd|jdd|jdkrtj|jdt|j t d|j itj |jdt|j dkrft|jd}|dkr||_d|_nZ|tkr|jd}tj}||jd|}td||jddnd|_|dSt|jtt|}|j|j|j|}|t|d|<|j|8_|j|7_|S) Nz,No buffer for stream in _ReadStream.readintorrZgetMorez timed-outr Exception thrown in fetch: F)r0r&rr7r+Atomicsstore ERROR_TIMEOUTr3rBr,r4waitr2r(r6ERROR_EXCEPTION TextDecodernewdecoder1slicer8r<minlen memoryviewsubarrayto_py)rrHdata_len string_len js_decoderjson_strZ ret_lengthrVr r r!readintosN    z_ReadStream.readinto)r"r#r$rr>r@propertyrAr<rDrFrGr\r%r r rr!r-\s r-c@s*eZdZddddZddddd Zd S) _StreamingFetcherr:r;csdd_tjtgtddi}ddddfdd }tj|}tjj |_ tjj |_ dS) NFtypezapplication/javascriptr r:) js_resolve_fn js_reject_fnr*cs>dddfdd }dddfdd }|j_|j_dS)Nr r:)er*csd_|dSrC)streaming_readyrb)r`rr r!onMsgszC_StreamingFetcher.__init__..promise_resolver..onMsgcs |dSrr rd)rar r!onErrszC_StreamingFetcher.__init__..promise_resolver..onErr) js_workerZ onmessageonerror)r`rarerfr=)rar`r!promise_resolversz4_StreamingFetcher.__init__..promise_resolver) rcr+BlobrP_STREAMING_WORKER_CODEr,URLZcreateObjectURLZ globalThisWorkerrgZPromisejs_worker_ready_promise)rZ js_data_blobriZ js_data_urlr r=r!rs  z_StreamingFetcher.__init__r rrr*c Csdd|jD}|j}|t||jd}|jdkrDtd|jnd}tj d}tj |}tj |d}tj |dttj |dtj |jtjj} |jt|| |dtj |dt||dtkrtd |dd n|dtkrP|d } tj } | |d| } t| } t|| d | d t |||j|j| d|dS|dt!kr|d } tj } | |d| } t"d| |dd nt"d|d|dd dS)NcSsi|]\}}|tkr||qSr )HEADERS_TO_IGNORE).0kvr r r! sz*_StreamingFetcher.send..)headersbodymethodrr5i)bufferurlZ fetchParamsz'Timeout connecting to streaming requestrr statusruZ connectionID)r status_coderurvrIz%Unknown status from worker in fetch: )#ruitemsrvr rwr2r/r+ZSharedArrayBufferrPZ Int32ArrayZ Uint8ArrayrJrKrLnotifyrlrzlocationhrefrgrBr,rMr(SUCCESS_HEADERrOrQrRjsonloadsrr-rNr&)rrrurvZ fetch_datar2Zjs_shared_bufferZ js_int_bufferZjs_byte_bufferZjs_absolute_urlrYrZr[Z response_objr r r!sendsv         z_StreamingFetcher.sendN)r"r#r$rrr r r r!r^sr^r?r;cCs ttdottdotjtjkS)Nwindowr)hasattrr+rrr r r r!is_in_browser_main_thread$srcCsttdotjS)NcrossOriginIsolated)rr+rr r r r!is_cross_origin_isolated(srcCs2ttdo0ttjdo0ttjjdo0tjjjdkS)Nprocessreleasenamenode)rr+rrrr r r r! is_in_node,s    rcCsttdottdS)Nrmrj)rr+r r r r!is_worker_available5srz_StreamingFetcher | None_fetcherr rrocCs"trtrt|StdSdSr)rrcr_show_streaming_warningrr r r!send_streaming_requestDs  rFr:cCstsdad}tj|dS)NTz8Warning: Timeout is not available on main browser thread)_SHOWN_TIMEOUT_WARNINGr+consolewarn)rr r r!_show_timeout_warningOsrcCsbts^dad}ts|d7}tr(|d7}ts6|d7}tdkrH|d7}dd lm}||dS) NTz%Can't stream HTTP requests because: z$ Page is not cross-origin isolated z+ Python is running in main browser thread z> Worker or Blob classes are not available in this environment.Fz Streaming fetch worker isn't ready. If you want to be sure that streaming fetch is working, you need to call: 'await urllib3.contrib.emscripten.fetch.wait_for_streaming_ready()`r)r)_SHOWN_STREAMING_WARNINGrrrrcr+rr)rrr r r!rZs  rrc CsLztj}ts0d|_|jrFt|jd|_n|d|jrFt| |j |j d|j D] \}}|tkrb|||qb|t|jtt|}ts|j}n |jd}t|j|||dWStk rF}zJ|jdkr t |j!|dn*|jd kr(t"|j!|dnt"|j!|dW5d}~XYnXdS) NZ arraybufferr5ztext/plain; charset=ISO-8859-15Fz ISO-8859-15)r|rurvr TimeoutErrorrZ NetworkError)#r+ZXMLHttpRequestrPrZ responseTyper2r/ZoverrideMimeTyperopenrwrzrur}lowerrpZsetRequestHeaderrr rvdictrparsestrZgetAllResponseHeadersrrWtobytesencoderr{rrr(rr)rZjs_xhrrvaluerurverrr r r! send_requestms<      rz bool | NonecCstr tjSdSdSr)rrcr r r r!rcsrccstrtjIdHdSdSdS)NTF)rrnr r r r!wait_for_streaming_readys r)7__doc__ __future__rior email.parserrimportlib.resourcesrtypingrrr+Z pyodide.ffirrr r Ztyping_extensionsr rr rrrprZ SUCCESS_EOFrLrN __package__joinpath read_textrk Exceptionrr&r(r, RawIOBaser-r^rrrrr__annotations__rrrrrrrcrr r r r!sf      fb  )