a DOg+@s`ddgZddlZddlZddlZddlZddlZddlZddlZddlZddl Z ddl m Z ddl m Z m Z ddlmZdZd Zd Zd ZeZd d ZddZGdddeZGdddZddZGdddeZd*ddZddZGdddeZ Gd dde!Z"Gd!d"d"e!Z#e#Z$Gd#d$d$e#Z%Gd%d&d&e!Z&Gd'd(d(e&Z'Gd)dde"Z(dS)+Pool ThreadPoolN)util) get_context TimeoutError)waitINITRUNCLOSE TERMINATEcCs tt|SN)listmapargsr9/opt/alt/python39/lib64/python3.9/multiprocessing/pool.pymapstar/srcCstt|d|dS)Nrr)r itertoolsstarmaprrrr starmapstar2src@seZdZddZddZdS)RemoteTracebackcCs ||_dSr tb)selfrrrr__init__:szRemoteTraceback.__init__cCs|jSr rrrrr__str__<szRemoteTraceback.__str__N)__name__ __module__ __qualname__rrrrrrr9src@seZdZddZddZdS)ExceptionWithTracebackcCs0tt|||}d|}||_d||_dS)Nz """ %s""") tracebackformat_exceptiontypejoinexcr)rr(rrrrr@s zExceptionWithTraceback.__init__cCst|j|jffSr ) rebuild_excr(rrrrr __reduce__Esz!ExceptionWithTraceback.__reduce__N)rr r!rr*rrrrr"?sr"cCst||_|Sr )r __cause__)r(rrrrr)Hs r)cs,eZdZfddZddZddZZS)MaybeEncodingErrorcs.t||_t||_tt||j|jdSr )reprr(valuesuperr,r)rr(r. __class__rrrTs  zMaybeEncodingError.__init__cCsd|j|jfS)Nz(Error sending result: '%s'. Reason: '%s')r.r(rrrrrYszMaybeEncodingError.__str__cCsd|jj|fS)Nz<%s: %s>)r1rrrrr__repr__]szMaybeEncodingError.__repr__)rr r!rrr2 __classcell__rrr0rr,Ps r,rFc Cs|dur(t|tr|dks(td||j}|j}t|drR|j|j |durb||d}|dus~|r||krz |} Wn&t t fyt dYqYn0| durt dq| \} } } } }zd| | i|f}WnJty4}z0|r| turt||j}d|f}WYd}~n d}~00z|| | |fWnTty}z:t||d}t d ||| | d|ffWYd}~n d}~00d} } }} } }|d7}qft d |dS) NrzMaxtasks {!r} is not valid_writerrz)worker got EOFError or OSError -- exitingzworker got sentinel -- exitingTFz0Possible encoding error while sending result: %szworker exiting after %d tasks) isinstanceintAssertionErrorformatputgethasattrr4close_readerEOFErrorOSErrorrdebug Exception_helper_reraises_exceptionr" __traceback__r,)inqueueoutqueue initializerinitargsZmaxtaskswrap_exceptionr9r:Z completedtaskjobifuncrkwdsresultewrappedrrrworkerasN        ( rQcCs|dSr r)ZexrrrrBsrBcs.eZdZddfdd ZfddZZS) _PoolCacheNnotifiercs||_tj|i|dSr )rTr/r)rrTrrMr0rrrsz_PoolCache.__init__cs t||s|jddSr )r/ __delitem__rTr9)ritemr0rrrUs z_PoolCache.__delitem__)rr r!rrUr3rrr0rrRsrRc@seZdZdZeddZdKddZeje fdd Z d d Z d d Z eddZ eddZddZeddZeddZddZddZdifddZdLddZdMd d!ZdNd"d#Zd$d%ZdOd'd(ZdPd)d*Zdiddfd+d,ZdQd-d.ZdRd/d0ZedSd1d2Zed3d4Z ed5d6Z!ed7d8Z"ed9d:Z#d;d<Z$d=d>Z%d?d@Z&dAdBZ'edCdDZ(edEdFZ)dGdHZ*dIdJZ+dS)TrTcOs|j|i|Sr Process)ctxrrMrrrrXsz Pool.ProcessNrcCsg|_t|_|pt|_|t|_|j|_ t |j d|_ ||_ ||_ ||_|durjtphd}|dkrztd|durt|std||_z |WnFty|jD]}|jdur|q|jD] }|qւYn0|}tjtj|j |j|j|j|j|j|j |j!|j |j|j |j"||j fd|_#d|j#_$t%|j#_|j#&tjtj'|j|j(|j!|j|j fd|_)d|j)_$t%|j)_|j)&tjtj*|j!|j+|j fd|_,d|j,_$t%|j,_|j,&t-j.||j/|j|j |j!|j|j |j#|j)|j,|j f dd|_0t%|_dS) NrSrz&Number of processes must be at least 1zinitializer must be a callabletargetrT)rZ exitpriority)1_poolr _stater_ctx _setup_queuesqueue SimpleQueue _taskqueue_change_notifierrR_cache_maxtasksperchild _initializer _initargsos cpu_count ValueErrorcallable TypeError _processes_repopulate_poolrAexitcode terminater'_get_sentinels threadingZThreadr_handle_workersrX_inqueue _outqueue_wrap_exception_worker_handlerdaemonr start _handle_tasks _quick_put _task_handler_handle_results _quick_get_result_handlerrZFinalize_terminate_pool _terminate)r processesrFrGmaxtasksperchildcontextp sentinelsrrrrs~                  z Pool.__init__cCs>|j|kr:|d|t|dt|dddur:|jddS)Nz&unclosed running multiprocessing pool )sourcerd)r^ResourceWarninggetattrrdr9)rZ_warnr rrr__del__s   z Pool.__del__c Cs0|j}d|jd|jd|jdt|jd S)N<.z state=z pool_size=>)r1r r!r^lenr])rclsrrrr2s z Pool.__repr__cCs |jjg}|jjg}g||Sr )rvr=rd)rZtask_queue_sentinelsZself_notifier_sentinelsrrrrrs  zPool._get_sentinelscCsdd|DS)NcSsg|]}t|dr|jqS)sentinel)r;r).0rQrrr s z.Pool._get_worker_sentinels..rZworkersrrr_get_worker_sentinelsszPool._get_worker_sentinelscCsPd}ttt|D]6}||}|jdurtd||d}||=q|S)NFcleaning up worker %dT)reversedrangerrprr@r')poolZcleanedrKrQrrr_join_exited_workerss zPool._join_exited_workersc Cs0||j|j|j|j|j|j|j|j|j |j Sr ) _repopulate_pool_staticr_rXrnr]rurvrgrhrfrwrrrrro.s zPool._repopulate_poolc Csft|t|D]P} ||t|||||| fd} | jdd| _d| _| || t dqdS)NrZrXZ PoolWorkerTz added worker) rrrQnamereplaceryrzappendrr@) rYrXrrrDrErFrGrrHrKwrrrr7s zPool._repopulate_pool_staticc Cs*t|r&t|||||||||| dSr )rrr) rYrXrrrDrErFrGrrHrrr_maintain_poolJs   zPool._maintain_poolcCs4|j|_|j|_|jjj|_|jjj|_ dSr ) r_rbrurvr4sendr|r=recvrrrrrr`Vs   zPool._setup_queuescCs|jtkrtddS)NzPool not running)r^r rkrrrr_check_running\s zPool._check_runningcCs||||Sr ) apply_asyncr:)rrLrrMrrrapply`sz Pool.applycCs|||t|Sr ) _map_asyncrr:rrLiterable chunksizerrrrgszPool.mapcCs|||t|Sr )rrr:rrrrrnsz Pool.starmapcCs|||t|||Sr )rrrrLrrcallbackerror_callbackrrr starmap_asyncvs zPool.starmap_asyncc cslz,d}t|D]\}}||||fifVqWn:tyf}z"||dt|fifVWYd}~n d}~00dS)Nr) enumeraterArB)rZ result_jobrLrrKxrOrrr_guarded_task_generation~s zPool._guarded_task_generationrcCs||dkr:t|}|j||j|||jf|S|dkrPtd|t |||}t|}|j||jt ||jfdd|DSdS)NrzChunksize must be 1+, not {0:n}css|]}|D] }|Vq qdSr rrchunkrVrrr zPool.imap..) r IMapIteratorrcr9r_job _set_lengthrkr8r _get_tasksrrrLrrrN task_batchesrrrimaps4z Pool.imapcCs||dkr:t|}|j||j|||jf|S|dkrPtd|t |||}t|}|j||jt ||jfdd|DSdS)NrzChunksize must be 1+, not {0!r}css|]}|D] }|Vq qdSr rrrrrrrz&Pool.imap_unordered..) rIMapUnorderedIteratorrcr9rrrrkr8rrrrrrrimap_unordereds0zPool.imap_unorderedcCs6|t|||}|j|jd|||fgdf|SNr)r ApplyResultrcr9r)rrLrrMrrrNrrrrs zPool.apply_asynccCs|||t|||Sr )rrrrrr map_asyncszPool.map_asyncc Cs|t|dst|}|durJtt|t|jd\}}|rJ|d7}t|dkrZd}t|||}t||t|||d} |j | | j ||df| S)N__len__rrr) rr;rdivmodrr]rr MapResultrcr9rr) rrLrZmapperrrrZextrarrNrrrrs,  zPool._map_asynccCs"t||d|s|q dS)N)timeout)remptyr:)rchange_notifierrrrr_wait_for_updatess zPool._wait_for_updatesc Cstt}|jtks |r\|jtkr\|||||||| | | | g||| }|||q|dt ddS)Nzworker handler exiting) rscurrent_threadr^r r rrrr9rr@)rcache taskqueuerYrXrrrDrErFrGrrHrrthreadZcurrent_sentinelsrrrrts zPool._handle_workersc Cst}t|jdD] \}}d}z|D]}|jtkrDtdqz ||Wq(ty} zH|dd\} } z||  | d| fWnt yYn0WYd} ~ q(d} ~ 00q(|rtd|r|dnd} || dWd}}} qWd}}} q,Wd}}} qd}}} 0qtdz6td| dtd |D]} |dqPWnt ytd Yn0td dS) Nz'task handler found thread._state != RUNFzdoing set_length()rrztask handler got sentinelz/task handler sending sentinel to result handlerz(task handler sending sentinel to workersz/task handler got OSError when sending sentinelsztask handler exiting) rsriterr:r^r rr@rA_setKeyErrorr9r?) rr9rErrrZtaskseqZ set_lengthrIrOrJidxrrrrr{ sJ            zPool._handle_tasksc Cst}z |}Wn"ttfy4tdYdS0|jtkrLtdq|dur`tdq|\}}}z||||Wnt yYn0d}}}q|r6|jt kr6z |}Wn"ttfytdYdS0|durtdq|\}}}z||||Wnt y&Yn0d}}}qt |drtdz,t dD]}|j slqv|qVWnttfyYn0tdt||jdS) Nz.result handler got EOFError/OSError -- exitingz,result handler found thread._state=TERMINATEzresult handler got sentinelz&result handler ignoring extra sentinelr=z"ensuring that outqueue is not full z7result handler exiting: len(cache)=%s, thread._state=%s)rsrr?r>rr@r^r rrr r;rr=pollr)rEr:rrrIrJrKobjrrrr~:sZ                zPool._handle_resultsccs0t|}tt||}|s dS||fVqdSr )rtuplerislice)rLitsizerrrrrvs zPool._get_taskscCs tddS)Nz:pool objects cannot be passed between processes or pickled)NotImplementedErrorrrrrr*szPool.__reduce__cCs2td|jtkr.t|_t|j_|jddS)Nz closing pool)rr@r^r r rxrdr9rrrrr<s   z Pool.closecCstdt|_|dS)Nzterminating pool)rr@r r^rrrrrrqs zPool.terminatecCsjtd|jtkrtdn|jttfvr4td|j|j |j |j D] }|qXdS)Nz joining poolzPool is still runningzIn unknown state) rr@r^r rkr r rxr'r}rr])rrrrrr's       z Pool.joincCs@td|j|r<|jr<|jt dqdS)Nz7removing tasks from inqueue until task handler finishedr) rr@Z_rlockacquireis_aliver=rrtimesleep)rD task_handlerrrrr_help_stuff_finishs    zPool._help_stuff_finishc CsXtdt|_|dt|_td|||t||sXt| dkrXtdt|_|d|dtdt |ur| |rt |ddrtd|D]} | j dur| qtdt |ur| td t |ur| |rTt |ddrTtd |D](} | r*td | j| q*dS) Nzfinalizing poolz&helping task handler/workers to finishrz.Cannot have cache with result_hander not alivezjoining worker handlerrqzterminating workerszjoining task handlerzjoining result handlerzjoining pool workersr)rr@r r^r9rrrr7rsrr'r;rprqpid) rrrDrErrZworker_handlerrZresult_handlerrrrrrrsB               zPool._terminate_poolcCs ||Sr )rrrrr __enter__szPool.__enter__cCs |dSr )rq)rexc_typeZexc_valZexc_tbrrr__exit__sz Pool.__exit__)NNrNN)N)N)NNN)r)r)NNN)NNN)N),rr r!rw staticmethodrXrwarningswarnr rr2rrrrrorrr`rrrrrrrrrrrr classmethodrtr{r~rr*r<rqr'rrrrrrrrrst  P                - ;    5c@sJeZdZddZddZddZddd Zdd d Zd d Ze e j Z dS)rcCs>||_t|_tt|_|j|_||_||_ ||j|j<dSr ) r]rsZEvent_eventnext job_counterrre _callback_error_callback)rrrrrrrrs  zApplyResult.__init__cCs |jSr )rZis_setrrrrreadyszApplyResult.readycCs|std||jS)Nz{0!r} not ready)rrkr8_successrrrr successfulszApplyResult.successfulNcCs|j|dSr )rrrrrrrrszApplyResult.waitcCs,|||st|jr"|jS|jdSr )rrrr_valuerrrrr:s  zApplyResult.getcCsZ|\|_|_|jr$|jr$||j|jr<|js<||j|j|j|j=d|_dSr ) rrrrrsetrerr]rrKrrrrrs       zApplyResult._set)N)N) rr r!rrrrr:rrtypes GenericAlias__class_getitem__rrrrrs    rc@seZdZddZddZdS)rcCshtj||||dd|_dg||_||_|dkrNd|_|j|j|j =n||t |||_dS)NrTr) rrrr _chunksize _number_leftrrrerbool)rrrlengthrrrrrrs    zMapResult.__init__cCs|jd8_|\}}|rv|jrv||j||j|d|j<|jdkr|jrZ||j|j|j=|jd|_ nL|s|jrd|_||_|jdkr|j r| |j|j|j=|jd|_ dS)NrrF) rrrrrrerrrr]r)rrKZsuccess_resultsuccessrNrrrr&s&          zMapResult._setN)rr r!rrrrrrrs rc@s:eZdZddZddZd ddZeZdd Zd d ZdS) rcCsT||_tt|_tt|_|j|_t |_ d|_ d|_ i|_||j|j<dSr)r]rsZ ConditionZLock_condrrrre collectionsdeque_items_index_length _unsorted)rrrrrrDs  zIMapIterator.__init__cCs|Sr rrrrr__iter__OszIMapIterator.__iter__Nc Cs|jz|j}Wnvty|j|jkr sN    -=++E