o ?Og@sdddgZddlZddlZddlZddlZddlZddlZddlZddlZddl Z ddl m Z ddl m Z m Z ddlmZdZd Zd Zd ZeZd d ZddZGdddeZGdddZddZGdddeZ  d*ddZddZGdddeZ Gd dde!Z"Gd!d"d"e!Z#e#Z$Gd#d$d$e#Z%Gd%d&d&e!Z&Gd'd(d(e&Z'Gd)dde"Z(dS)+Pool ThreadPoolN)util) get_context TimeoutError)waitINITRUNCLOSE TERMINATEcCs tt|SN)listmapargsr;/opt/alt/python310/lib64/python3.10/multiprocessing/pool.pymapstar/ rcCstt|d|dS)Nrr)r itertoolsstarmaprrrr starmapstar2src@eZdZddZddZdS)RemoteTracebackcCs ||_dSr tb)selfrrrr__init__: zRemoteTraceback.__init__cCs|jSr rrrrr__str__<szRemoteTraceback.__str__N)__name__ __module__ __qualname__rr!rrrrr9s rc@r)ExceptionWithTracebackcCs0tt|||}d|}||_d||_dS)Nz """ %s""") tracebackformat_exceptiontypejoinexcr)rr+rrrrr@s zExceptionWithTraceback.__init__cCst|j|jffSr ) rebuild_excr+rr rrr __reduce__Ez!ExceptionWithTraceback.__reduce__N)r"r#r$rr-rrrrr%?s r%cCst||_|Sr )r __cause__)r+rrrrr,Hs r,cs.eZdZ fddZddZddZZS)MaybeEncodingErrorcs.t||_t||_tt||j|jdSr )reprr+valuesuperr0r)rr+r2 __class__rrrTs  zMaybeEncodingError.__init__cCsd|j|jfS)Nz(Error sending result: '%s'. Reason: '%s')r2r+r rrrr!YszMaybeEncodingError.__str__cCsd|jj|fS)Nz<%s: %s>)r5r"r rrr__repr__]r.zMaybeEncodingError.__repr__)r"r#r$rr!r6 __classcell__rrr4rr0Ps  r0rFc Cs|durt|tr |dkstd||j}|j}t|dr)|j|j |dur1||d}|dus=|r||krz|} Wnt t fyRt dYnw| dur]t dn| \} } } } }z d| | i|f}Wn"ty}z|r| turt||j}d|f}WYd}~nd}~wwz || | |fWn)ty}zt||d}t d ||| | d|ffWYd}~nd}~wwd} } }} } }|d7}|dus=|r||ks=t d |dS) NrzMaxtasks {!r} is not valid_writerrz)worker got EOFError or OSError -- exitingzworker got sentinel -- exitingTFz0Possible encoding error while sending result: %szworker exiting after %d tasks) isinstanceintAssertionErrorformatputgethasattrr8close_readerEOFErrorOSErrorrdebug Exception_helper_reraises_exceptionr% __traceback__r0)inqueueoutqueue initializerinitargsZmaxtaskswrap_exceptionr=r>Z completedtaskjobifuncrkwdsresultewrappedrrrworkerasX        rUcCs |r r)ZexrrrrFsrFcs0eZdZ ddfdd ZfddZZS) _PoolCacheNnotifiercs||_tj|i|dSr )rXr3r)rrXrrQr4rrrsz_PoolCache.__init__cs$t||s|jddSdSr )r3 __delitem__rXr=)ritemr4rrrYs z_PoolCache.__delitem__)r"r#r$rrYr7rrr4rrVsrVc@seZdZ dZeddZ  dKddZeje fdd Z d d Z d d Z eddZ eddZddZeddZeddZddZddZdifddZdLddZdLd d!Z  dMd"d#Zd$d%ZdNd'd(ZdNd)d*Zdiddfd+d,Z  dMd-d.Z  dMd/d0ZedLd1d2Zed3d4Z ed5d6Z!ed7d8Z"ed9d:Z#d;d<Z$d=d>Z%d?d@Z&dAdBZ'edCdDZ(edEdFZ)dGdHZ*dIdJZ+dS)OrTcOs|j|i|Sr Process)ctxrrQrrrr\sz Pool.ProcessNrcCs0g|_t|_|p t|_|t|_|j|_ t |j d|_ ||_ ||_ ||_|dur5tp4d}|dkr=td|durNt|trJ|dkrNtd|durZt|sZtd||_z|Wn!ty|jD] }|jdurx|qm|jD]}|q|w|}tjtj |j |j|j|j!|j|j|j"|j#|j |j|j |j$||j fd|_%d|j%_&t'|j%_|j%(tjtj)|j|j*|j#|j|j fd|_+d|j+_&t'|j+_|j+(tjtj,|j#|j-|j fd|_.d|j._&t'|j._|j.(t/j0||j1|j|j"|j#|j|j |j%|j+|j.|j f d d |_2t'|_dS) NrWrz&Number of processes must be at least 1rz/maxtasksperchild must be a positive int or Nonezinitializer must be a callabletargetrT)rZ exitpriority)3_poolr _stater_ctx _setup_queuesqueue SimpleQueue _taskqueue_change_notifierrV_cache_maxtasksperchild _initializer _initargsos cpu_count ValueErrorr9r:callable TypeError _processes_repopulate_poolrEexitcode terminater*_get_sentinels threadingZThreadr_handle_workersr\_inqueue _outqueue_wrap_exception_worker_handlerdaemonr start _handle_tasks _quick_put _task_handler_handle_results _quick_get_result_handlerrZFinalize_terminate_pool _terminate)r processesrJrKmaxtasksperchildcontextp sentinelsrrrrs                  z Pool.__init__cCsF|j|kr|d|t|dt|dddur!|jddSdSdS)Nz&unclosed running multiprocessing pool )sourcerh)rbResourceWarninggetattrrhr=)rZ_warnr rrr__del__ s  z Pool.__del__c Cs0|j}d|jd|jd|jdt|jd S)N<.z state=z pool_size=>)r5r#r$rblenra)rclsrrrr6s z Pool.__repr__cCs |jjg}|jjg}g||Sr )rzrArh)rZtask_queue_sentinelsZself_notifier_sentinelsrrrrvs   zPool._get_sentinelscCsdd|DS)NcSsg|] }t|dr|jqS)sentinel)r?r).0rUrrr s z.Pool._get_worker_sentinels..rZworkersrrr_get_worker_sentinelsszPool._get_worker_sentinelscCsR d}ttt|D]}||}|jdur&td||d}||=q |S)NFcleaning up worker %dT)reversedrangerrtrrDr*)poolZcleanedrOrUrrr_join_exited_workers!s zPool._join_exited_workersc Cs0||j|j|j|j|j|j|j|j|j |j Sr ) _repopulate_pool_staticrcr\rrraryrzrkrlrjr{r rrrrs1s zPool._repopulate_poolc Csh t|t|D](} ||t|||||| fd} | jdd| _d| _| || t dq dS)Nr^r\Z PoolWorkerTz added worker) rrrUnamereplacer}r~appendrrD) r]r\rrrHrIrJrKrrLrOwrrrr:s  zPool._repopulate_pool_staticc Cs0 t|rt|||||||||| dSdSr )rrr) r]r\rrrHrIrJrKrrLrrr_maintain_poolMs  zPool._maintain_poolcCs4|j|_|j|_|jjj|_|jjj|_ dSr ) rcrfryrzr8sendrrArecvrr rrrrdYs   zPool._setup_queuescCs|jtkr tddS)NzPool not running)rbr ror rrr_check_running_s zPool._check_runningcCs ||||Sr ) apply_asyncr>)rrPrrQrrrapplycsz Pool.applycC |||t|Sr ) _map_asyncrr>rrPiterable chunksizerrrrjszPool.mapcCrr )rrr>rrrrrqsz Pool.starmapcC |||t|||Sr )rrrrPrrcallbackerror_callbackrrr starmap_asyncys zPool.starmap_asyncc csp zd}t|D] \}}||||fifVq WdSty7}z||dt|fifVWYd}~dSd}~ww)Nr) enumeraterErF)rZ result_jobrPrrOxrSrrr_guarded_task_generations$zPool._guarded_task_generationrcC ||dkrt|}|j||j|||jf|S|dkr)td|t |||}t|}|j||jt ||jfdd|DS)NrzChunksize must be 1+, not {0:n}cs|] }|D]}|VqqdSr rrchunkrZrrr zPool.imap..) r IMapIteratorrgr=r_job _set_lengthror<r _get_tasksrrrPrrrR task_batchesrrrimaps6z Pool.imapcCr)NrzChunksize must be 1+, not {0!r}csrr rrrrrrrz&Pool.imap_unordered..) rIMapUnorderedIteratorrgr=rrrror<rrrrrrrimap_unordereds2zPool.imap_unorderedcCs8 |t|||}|j|jd|||fgdf|SNr)r ApplyResultrgr=r)rrPrrQrrrRrrrrs  zPool.apply_asynccCrr )rrrrrr map_asyncszPool.map_asyncc Cs |t|dst|}|dur&tt|t|jd\}}|r&|d7}t|dkr.d}t|||}t||t|||d} |j | | j ||df| S)N__len__rrr) rr?rdivmodrrarr MapResultrgr=rr) rrPrZmapperrrrZextrarrRrrrrs.  zPool._map_asynccCs,t||d|s||r dSdS)N)timeout)remptyr>)rchange_notifierrrrr_wait_for_updatess zPool._wait_for_updatesc Cst}|jtks|r9|jtkr9|||||||| | | | g||| }||||jtks|r9|jtks|dt ddS)Nzworker handler exiting) rwcurrent_threadrbr r rrrr=rrD)rcache taskqueuer]r\rrrHrIrJrKrrLrrthreadZcurrent_sentinelsrrrrxs  zPool._handle_workersc Cstt}t|jdD]z\}}d}zm|D]D}|jtkr!tdnTz||WqtyW} z$|dd\} } z ||  | d| fWn t yLYnwWYd} ~ qd} ~ ww|rmtd|re|dnd} || dWd}}} q Wd}}} n d}}} wtdztd| dtd |D]} |dqWnt ytd Ynwtd dS) Nz'task handler found thread._state != RUNFzdoing set_length()rrztask handler got sentinelz/task handler sending sentinel to result handlerz(task handler sending sentinel to workersz/task handler got OSError when sending sentinelsztask handler exiting) rwriterr>rbr rrDrE_setKeyErrorr=rC) rr=rIrrrZtaskseqZ set_lengthrMrSrNidxrrrrrsN            zPool._handle_tasksc Cst} z|}WnttfytdYdSw|jtkr'tdn*|dur1tdn |\}}}z ||||Wn t yIYnwd}}}q|r|jt krz|}WnttfyntdYdSw|durytdqQ|\}}}z ||||Wn t yYnwd}}}|r|jt ksXt |drtdzt dD] }|j sn|qWn ttfyYnwtd t||jdS) Nrz.result handler got EOFError/OSError -- exitingz,result handler found thread._state=TERMINATEzresult handler got sentinelz&result handler ignoring extra sentinelrAz"ensuring that outqueue is not full z7result handler exiting: len(cache)=%s, thread._state=%s)rwrrCrBrrDrbr rrr r?rrApollr)rIr>rrrMrNrOobjrrrr=sl                   zPool._handle_resultsccs0t|} tt||}|sdS||fVqr )rtuplerislice)rPitsizerrrrrys zPool._get_taskscCstd)Nz:pool objects cannot be passed between processes or pickled)NotImplementedErrorr rrrr-szPool.__reduce__cCs6td|jtkrt|_t|j_|jddSdS)Nz closing pool)rrDrbr r r|rhr=r rrrr@s  z Pool.closecCstdt|_|dS)Nzterminating pool)rrDr rbrr rrrrus  zPool.terminatecCshtd|jtkrtd|jttfvrtd|j|j |j |j D]}|q+dS)Nz joining poolzPool is still runningzIn unknown state) rrDrbr ror r r|r*rrra)rrrrrr*s       z Pool.joincCs\td|j|r(|jr,|jt d|r*|jsdSdSdSdS)Nz7removing tasks from inqueue until task handler finishedr) rrDZ_rlockacquireis_aliverArrtimesleep)rH task_handlerrrrr_help_stuff_finishs    "zPool._help_stuff_finishc CsVtdt|_|dt|_td|||t||s,t| dkr,tdt|_|d|dtdt |urH| |rdt |ddrdtd|D] } | j durc| qXtdt |urs| td t |ur| |rt |ddrtd |D]} | rtd | j| qdSdSdS) Nzfinalizing poolz&helping task handler/workers to finishrz.Cannot have cache with result_hander not alivezjoining worker handlerruzterminating workerszjoining task handlerzjoining result handlerzjoining pool workersr)rrDr rbr=rrrr;rwrr*r?rtrupid) rrrHrIrrZworker_handlerrZresult_handlerrrrrrrsJ              zPool._terminate_poolcCs ||Sr )rr rrr __enter__szPool.__enter__cCs |dSr )ru)rexc_typeZexc_valZexc_tbrrr__exit__rz Pool.__exit__)NNrNNr )NNN)r),r"r#r$r{ staticmethodr\rwarningswarnr rr6rvrrrsrrrdrrrrrrrrrrrr classmethodrxrrrr-r@rur*rrrrrrrrrsx  S                - ;   5 c@sJeZdZddZddZddZddd Zdd d Zd d Ze e j Z dS)rcCs>||_t|_tt|_|j|_||_||_ ||j|j<dSr ) rarwZEvent_eventnext job_counterrri _callback_error_callback)rrrrrrrrs  zApplyResult.__init__cCs |jSr )rZis_setr rrrreadyrzApplyResult.readycCs|s td||jS)Nz{0!r} not ready)rror<_successr rrr successfulszApplyResult.successfulNcCs|j|dSr )rrrrrrrrr.zApplyResult.waitcCs(|||s t|jr|jS|jr )rrrr_valuerrrrr>s zApplyResult.getcCsZ|\|_|_|jr|jr||j|jr|js||j|j|j|j=d|_dSr ) rrrrrsetrirrarrOrrrrrs        zApplyResult._setr ) r"r#r$rrrrr>rrtypes GenericAlias__class_getitem__rrrrrs     rc@r)rcCsjtj||||dd|_dg||_||_|dkr(d|_|j|j|j =dS||t |||_dS)NrTr) rrrr _chunksize _number_leftrrrirbool)rrrlengthrrrrrrs   zMapResult.__init__cCs|jd8_|\}}|r>|jr>||j||j|d|j<|jdkr<|jr-||j|j|j=|jd|_ dSdS|sI|jrId|_||_|jdkrf|j rW| |j|j|j=|jd|_ dSdS)NrrF) rrrrrrirrrrar)rrOZsuccess_resultsuccessrRrrrr)s*            zMapResult._setN)r"r#r$rrrrrrrs rc@s:eZdZddZddZd ddZeZdd Zd d ZdS) rcCsT||_tt|_tt|_|j|_t |_ d|_ d|_ i|_||j|j<dSr)rarwZ ConditionZLock_condrrrri collectionsdeque_items_index_length _unsorted)rrrrrrGs  zIMapIterator.__init__cCs|Sr rr rrr__iter__RszIMapIterator.__iter__Nc Cs|jIz|j}Wn9tyD|j|jkrd|_td|j|z|j}WntyA|j|jkr>d|_tdt dwYnwWdn1sOwY|\}}|r\|S|r ) rr popleft IndexErrorr r ra StopIterationrr)rrrZrr2rrrrUs0     zIMapIterator.nextcCs|j\|j|kr<|j||jd7_|j|jvr6|j|j}|j||jd7_|j|jvs|jn||j|<|j|jkrW|j|j =d|_ WddSWddS1sbwYdSNr) rr r rr popnotifyr rirrarrrrrms"         "zIMapIterator._setcCsh|j'||_|j|jkr"|j|j|j=d|_WddSWddS1s-wYdSr )rr r rrirra)rrrrrr~s   "zIMapIterator._set_lengthr ) r"r#r$rr r__next__rrrrrrrEs  rc@seZdZddZdS)rcCs||j1|j||jd7_|j|j|jkr,|j|j=d|_WddSWddS1s7wYdSr) rr rr rr rirrarrrrrs    "zIMapUnorderedIterator._setN)r"r#r$rrrrrrs rc@sVeZdZdZeddZdddZdd Zd d Zed d Z eddZ ddZ dS)rFcOsddlm}||i|S)Nrr[)Zdummyr\)r]rrQr\rrrr\s zThreadPool.ProcessNrcCst||||dSr )rr)rrrJrKrrrrszThreadPool.__init__cCs,t|_t|_|jj|_|jj|_dSr )rerfryrzr=rr>rr rrrrds   zThreadPool._setup_queuescCs |jjgSr )rhrAr rrrrvrzThreadPool._get_sentinelscCsgSr rrrrrrsz ThreadPool._get_worker_sentinelscCsBz |jddqtjyYnwt|D]}|dqdS)NTF)block)r>reZEmptyrr=)rHrrrOrrrrs   zThreadPool._help_stuff_finishcCst|dSr )rr)rrrrrrrrszThreadPool._wait_for_updates)NNr) r"r#r$r{rr\rrdrvrrrrrrrrs     )NrNF))__all__rrrmrerwrr'rrr&rrrZ connectionrr r r r countrrrrErr%r,r0rUrFdictrVobjectrrZ AsyncResultrrrrrrrrsP     -@++E