U e5d~@sdddgZddlZddlZddlZddlZddlZddlZddlZddlZddlm Z ddl m Z ddl m Z m Z ddlmZd Zd Zd Zd ZeZd dZddZGdddeZGdddZddZGdddeZd+ddZddZGdd d eZ Gd!dde!Z"Gd"d#d#e!Z#e#Z$Gd$d%d%e#Z%Gd&d'd'e!Z&Gd(d)d)e&Z'Gd*dde"Z(dS),Pool ThreadPoolN)Empty)util) get_context TimeoutError)waitINITRUNCLOSE TERMINATEcCs tt|SN)listmapargsr,/usr/lib64/python3.8/multiprocessing/pool.pymapstar/srcCstt|d|dS)Nrr)r itertoolsstarmaprrrr starmapstar2src@seZdZddZddZdS)RemoteTracebackcCs ||_dSrtb)selfrrrr__init__:szRemoteTraceback.__init__cCs|jSrrrrrr__str__<szRemoteTraceback.__str__N)__name__ __module__ __qualname__rrrrrrr9src@seZdZddZddZdS)ExceptionWithTracebackcCs0tt|||}d|}||_d||_dS)Nz """ %s""") tracebackformat_exceptiontypejoinexcr)rr)rrrrr@s zExceptionWithTraceback.__init__cCst|j|jffSr) rebuild_excr)rrrrr __reduce__Esz!ExceptionWithTraceback.__reduce__N)r r!r"rr+rrrrr#?sr#cCst||_|Sr)r __cause__)r)rrrrr*Hs r*cs,eZdZfddZddZddZZS)MaybeEncodingErrorcs.t||_t||_tt||j|jdSr)reprr)valuesuperr-r)rr)r/ __class__rrrTs  zMaybeEncodingError.__init__cCsd|j|jfS)Nz(Error sending result: '%s'. Reason: '%s')r/r)rrrrrYszMaybeEncodingError.__str__cCsd|jj|fS)Nz<%s: %s>)r2r rrrr__repr__]szMaybeEncodingError.__repr__)r r!r"rrr3 __classcell__rrr1rr-Ps r-rFc Cs|dk r(t|tr|dks(td||j}|j}t|drR|j|j |dk rb||d}|dks~|r||krz |} Wn(t t fk rt dYqYnX| dkrt dq| \} } } } }zd| | |f}WnHtk r0}z(|r| tk rt||j}d|f}W5d}~XYnXz|| | |fWnRtk r}z2t||d}t d ||| | d|ffW5d}~XYnXd} } }} } }|d7}qft d |dS) NrzMaxtasks {!r} is not valid_writerrz)worker got EOFError or OSError -- exitingzworker got sentinel -- exitingTFz0Possible encoding error while sending result: %szworker exiting after %d tasks) isinstanceintAssertionErrorformatputgethasattrr5close_readerEOFErrorOSErrorrdebug Exception_helper_reraises_exceptionr# __traceback__r-)inqueueoutqueue initializerinitargsZmaxtaskswrap_exceptionr:r;Z completedtaskjobifuncrkwdsresultewrappedrrrworkerasN        $ rRcCs|dSrr)ZexrrrrCsrCcs.eZdZddfdd ZfddZZS) _PoolCacheNnotifiercs||_tj||dSr)rUr0r)rrUrrNr1rrrsz_PoolCache.__init__cs t||s|jddSr)r0 __delitem__rUr:)ritemr1rrrVs z_PoolCache.__delitem__)r r!r"rrVr4rrr1rrSsrSc@seZdZdZeddZdKddZeje fdd Z d d Z d d Z eddZ eddZddZeddZeddZddZddZdifddZdLddZdMd d!ZdNd"d#Zd$d%ZdOd'd(ZdPd)d*Zdiddfd+d,ZdQd-d.ZdRd/d0ZedSd1d2Zed3d4Z ed5d6Z!ed7d8Z"ed9d:Z#d;d<Z$d=d>Z%d?d@Z&dAdBZ'edCdDZ(edEdFZ)dGdHZ*dIdJZ+dS)TrTcOs |j||SrProcess)ctxrrNrrrrYsz Pool.ProcessNrcCsg|_t|_|pt|_|t|_|j|_ t |j d|_ ||_ ||_ ||_|dkrjtphd}|dkrztd|dk rt|std||_z |WnHtk r|jD]}|jdkr|q|jD] }|q؂YnX|}tjtj|j |j|j|j|j|j|j |j!|j |j|j |j"||j fd|_#d|j#_$t%|j#_|j#&tjtj'|j|j(|j!|j|j fd|_)d|j)_$t%|j)_|j)&tjtj*|j!|j+|j fd|_,d|j,_$t%|j,_|j,&t-j.||j/|j|j |j!|j|j |j#|j)|j,|j f dd|_0t%|_dS) NrTrz&Number of processes must be at least 1zinitializer must be a callabletargetrT)rZ exitpriority)1_poolr _stater_ctx _setup_queuesqueue SimpleQueue _taskqueue_change_notifierrS_cache_maxtasksperchild _initializer _initargsos cpu_count ValueErrorcallable TypeError _processes_repopulate_poolrBexitcode terminater(_get_sentinels threadingZThreadr_handle_workersrY_inqueue _outqueue_wrap_exception_worker_handlerdaemonr start _handle_tasks _quick_put _task_handler_handle_results _quick_get_result_handlerrZFinalize_terminate_pool _terminate)r processesrGrHmaxtasksperchildcontextp sentinelsrrrrs              z Pool.__init__cCs>|j|kr:|d|t|dt|dddk r:|jddS)Nz&unclosed running multiprocessing pool )sourcere)r_ResourceWarninggetattrrer:)rZ_warnr rrr__del__s  z Pool.__del__c Cs0|j}d|jd|jd|jdt|jd S)N<.z state=z pool_size=>)r2r!r"r_lenr^)rclsrrrr3sz Pool.__repr__cCs|jjg}|jjg}||Sr)rwr>re)rZtask_queue_sentinelsZself_notifier_sentinelsrrrrss  zPool._get_sentinelscCsdd|DS)NcSsg|]}t|dr|jqS)sentinel)r<r).0rRrrr s z.Pool._get_worker_sentinels..rZworkersrrr_get_worker_sentinelsszPool._get_worker_sentinelscCsPd}ttt|D]6}||}|jdk rtd||d}||=q|S)NFcleaning up worker %dT)reversedrangerrqrrAr()poolZcleanedrLrRrrr_join_exited_workerss zPool._join_exited_workersc Cs0||j|j|j|j|j|j|j|j|j |j Sr) _repopulate_pool_staticr`rYror^rvrwrhrirgrxrrrrrp.s zPool._repopulate_poolc Csft|t|D]P} ||t|||||| fd} | jdd| _d| _| || t dqdS)Nr[rYZ PoolWorkerTz added worker) rrrRnamereplacerzr{appendrrA) rZrYrrrErFrGrHrrIrLwrrrr7s zPool._repopulate_pool_staticc Cs*t|r&t|||||||||| dSr)rrr) rZrYrrrErFrGrHrrIrrr_maintain_poolJs  zPool._maintain_poolcCs4|j|_|j|_|jjj|_|jjj|_ dSr) r`rcrvrwr5sendr}r>recvrrrrrraVs   zPool._setup_queuescCs|jtkrtddS)NzPool not running)r_r rlrrrr_check_running\s zPool._check_runningcCs||||Sr) apply_asyncr;)rrMrrNrrrapply`sz Pool.applycCs|||t|Sr) _map_asyncrr;rrMiterable chunksizerrrrgszPool.mapcCs|||t|Sr)rrr;rrrrrnsz Pool.starmapcCs|||t|||Sr)rrrrMrrcallbackerror_callbackrrr starmap_asyncvs zPool.starmap_asyncc csjz,d}t|D]\}}||||fifVqWn8tk rd}z||dt|fifVW5d}~XYnXdS)Nr) enumeraterBrC)rZ result_jobrMrrLxrPrrr_guarded_task_generation~s zPool._guarded_task_generationrcCs||dkr:t|}|j||j|||jf|S|dkrPtd|t |||}t|}|j||jt ||jfdd|DSdS)NrzChunksize must be 1+, not {0:n}css|]}|D] }|Vq qdSrrrchunkrWrrr szPool.imap..) r IMapIteratorrdr:r_job _set_lengthrlr9r _get_tasksrrrMrrrO task_batchesrrrimaps4z Pool.imapcCs||dkr:t|}|j||j|||jf|S|dkrPtd|t |||}t|}|j||jt ||jfdd|DSdS)NrzChunksize must be 1+, not {0!r}css|]}|D] }|Vq qdSrrrrrrrsz&Pool.imap_unordered..) rIMapUnorderedIteratorrdr:rrrrlr9rrrrrrrimap_unordereds0zPool.imap_unorderedcCs6|t|||}|j|jd|||fgdf|SNr)r ApplyResultrdr:r)rrMrrNrrrOrrrrs zPool.apply_asynccCs|||t|||Sr)rrrrrr map_asyncszPool.map_asyncc Cs|t|dst|}|dkrJtt|t|jd\}}|rJ|d7}t|dkrZd}t|||}t||t|||d} |j | | j ||df| S)N__len__rrr) rr<rdivmodrr^rr MapResultrdr:rr) rrMrZmapperrrrZextrarrOrrrrs,  zPool._map_asynccCs"t||d|s|q dS)N)timeout)r emptyr;)rchange_notifierrrrr_wait_for_updatess zPool._wait_for_updatesc Cspt}|jtks |rX|jtkrX|||||||| | | | ||| }|||q|dt ddS)Nzworker handler exiting) rtcurrent_threadr_r r rrrr:rrA)rcache taskqueuerZrYrrrErFrGrHrrIrrthreadZcurrent_sentinelsrrrrus zPool._handle_workersc Cspt}t|jdD]\}}d}z|D]}|jtkrBtdqz ||Wq&tk r} zB|dd\} } z||  | d| fWnt k rYnXW5d} ~ XYq&Xq&|rtd|r|dnd} || dWqW q W5d}}} Xqtdz6td| dtd |D]} |dq.Wn t k r`td YnXtd dS) Nz'task handler found thread._state != RUNFzdoing set_length()rrztask handler got sentinelz/task handler sending sentinel to result handlerz(task handler sending sentinel to workersz/task handler got OSError when sending sentinelsztask handler exiting) rtriterr;r_r rrArB_setKeyErrorr:r@) rr:rFrrrZtaskseqZ set_lengthrJrKrPidxrrrrr| sB         zPool._handle_tasksc Cst}z |}Wn$ttfk r6tdYdSX|jtkrNtdq|dkrbtdq|\}}}z||||Wnt k rYnXd}}}q|r@|jt kr@z |}Wn$ttfk rtdYdSX|dkrtdq|\}}}z||||Wnt k r0YnXd}}}qt |drtdz,t dD]}|j svq|q`Wnttfk rYnXtdt||jdS) Nz.result handler got EOFError/OSError -- exitingz,result handler found thread._state=TERMINATEzresult handler got sentinelz&result handler ignoring extra sentinelr>z"ensuring that outqueue is not full z7result handler exiting: len(cache)=%s, thread._state=%s)rtrr@r?rrAr_r rrr r<rr>pollr)rFr;rrrJrKrLobjrrrr:s\               zPool._handle_resultsccs0t|}tt||}|s dS||fVqdSr)rtuplerislice)rMitsizerrrrrvs zPool._get_taskscCs tddS)Nz:pool objects cannot be passed between processes or pickled)NotImplementedErrorrrrrr+szPool.__reduce__cCs2td|jtkr.t|_t|j_|jddS)Nz closing pool)rrAr_r r ryrer:rrrrr=s   z Pool.closecCstdt|_|dS)Nzterminating pool)rrAr r_rrrrrrrs zPool.terminatecCsjtd|jtkrtdn|jttfkr4td|j|j |j |j D] }|qXdS)Nz joining poolzPool is still runningzIn unknown state) rrAr_r rlr r ryr(r~rr^)rrrrrr(s       z Pool.joincCs@td|j|r<|jr<|jt dqdS)Nz7removing tasks from inqueue until task handler finishedr) rrAZ_rlockacquireis_aliver>rrtimesleep)rE task_handlerrrrr_help_stuff_finishs    zPool._help_stuff_finishc CsXtdt|_|dt|_td|||t||sXt| dkrXtdt|_|d|dtdt |k r| |rt |ddrtd|D]} | j dkr| qtdt |k r| td t |k r| |rTt |ddrTtd |D](} | r*td | j| q*dS) Nzfinalizing poolz&helping task handler/workers to finishrz.Cannot have cache with result_hander not alivezjoining worker handlerrrzterminating workerszjoining task handlerzjoining result handlerzjoining pool workersr)rrAr r_r:rrrr8rtrr(r<rqrrpid) rrrErFrrZworker_handlerrZresult_handlerrrrrrrsB               zPool._terminate_poolcCs ||Sr)rrrrr __enter__szPool.__enter__cCs |dSr)rr)rexc_typeZexc_valZexc_tbrrr__exit__sz Pool.__exit__)NNrNN)N)N)NNN)r)r)NNN)NNN)N),r r!r"rx staticmethodrYrwarningswarnr rr3rsrrrprrrarrrrrrrrrrrr classmethodrur|rrr+r=rrr(rrrrrrrrrsv  P                - ;    5c@s@eZdZddZddZddZddd Zdd d Zd d ZdS)rcCs>||_t|_tt|_|j|_||_||_ ||j|j<dSr) r^rtZEvent_eventnext job_counterrrf _callback_error_callback)rrrrrrrrs  zApplyResult.__init__cCs |jSr)rZis_setrrrrreadyszApplyResult.readycCs|std||jS)Nz{0!r} not ready)rrlr9_successrrrr successfulszApplyResult.successfulNcCs|j|dSr)rr rrrrrr szApplyResult.waitcCs,|||st|jr"|jS|jdSr)r rrr_valuerrrrr;s  zApplyResult.getcCsZ|\|_|_|jr$|jr$||j|jr<|js<||j|j|j|j=d|_dSr) rrrrrsetrfrr^rrLrrrrrs       zApplyResult._set)N)N) r r!r"rrrr r;rrrrrrs    rc@seZdZddZddZdS)rcCshtj||||dd|_dg||_||_|dkrNd|_|j|j|j =n||t |||_dS)NrTr) rrrr _chunksize _number_leftrrrfrbool)rrrlengthrrrrrrs    zMapResult.__init__cCs|jd8_|\}}|rv|jrv||j||j|d|j<|jdkr|jrZ||j|j|j=|jd|_ nL|s|jrd|_||_|jdkr|j r| |j|j|j=|jd|_ dS)NrrF) rrrrrrfrrrr^r)rrLZsuccess_resultsuccessrOrrrr$s&          zMapResult._setN)r r!r"rrrrrrrs rc@s:eZdZddZddZd ddZeZdd Zd d ZdS) rcCsT||_tt|_tt|_|j|_t |_ d|_ d|_ i|_||j|j<dSr)r^rtZ ConditionZLock_condrrrrf collectionsdeque_items_index_length _unsorted)rrrrrrBs  zIMapIterator.__init__cCs|Srrrrrr__iter__MszIMapIterator.__iter__Nc Cs|jz|j}Wnztk r|j|jkr>d|_td|j|z|j}Wn2tk r|j|jkrd|_tdt dYnXYnXW5QRX|\}}|r|S|dSr) rrpopleft IndexErrorrrr^ StopIterationr r)rrrWrr/rrrrPs&   zIMapIterator.nextc Cs|j|j|krn|j||jd7_|j|jkrb|j|j}|j||jd7_q,|jn ||j|<|j|jkr|j|j =d|_ W5QRXdSNr) rrrrrpopnotifyrrfrr^rrrrrhs        zIMapIterator._setc CsB|j2||_|j|jkr4|j|j|j=d|_W5QRXdSr)rrrrrfrr^)rrrrrrys    zIMapIterator._set_length)N) r r!r"rrr__next__rrrrrrr@s   rc@seZdZddZdS)rc CsV|jF|j||jd7_|j|j|jkrH|j|j=d|_W5QRXdSr) rrrrrrrfrr^rrrrrs    zIMapUnorderedIterator._setN)r r!r"rrrrrrsrc@sVeZdZdZeddZdddZdd Zd d Zed d Z eddZ ddZ dS)rFcOsddlm}|||S)NrrX)ZdummyrY)rZrrNrYrrrrYs zThreadPool.ProcessNrcCst||||dSr)rr)rrrGrHrrrrszThreadPool.__init__cCs,t|_t|_|jj|_|jj|_dSr)rbrcrvrwr:r}r;rrrrrras   zThreadPool._setup_queuescCs |jjgSr)rer>rrrrrsszThreadPool._get_sentinelscCsgSrrrrrrrsz ThreadPool._get_worker_sentinelscCsFz|jddqWntjk r(YnXt|D]}|dq2dS)NF)block)r;rbrrr:)rErrrLrrrrs  zThreadPool._help_stuff_finishcCst|dSr)rr)rrrrrrrrszThreadPool._wait_for_updates)NNr) r r!r"rxrrYrrarsrrrrrrrrs    )NrNF))__all__rrrjrbrtrr%rrr$rrrZ connectionr r r r r countrrrrBrr#r*r-rRrCdictrSobjectrrZ AsyncResultrrrrrrrr sN     -=)+E