B \NM@sdZddlZddlZddlmZddlZddlmZddlZddl m Z ddl m Z ddl Z ddlZddlmZddlZddlZddlZeZdaGd d d Zd d Zd ZdZGdddeZGdddZddZGdddeZ GdddeZ!GdddeZ"Gddde Z#ddZ$dd Z%d1d!d"Z&d#d$Z'd%d&Z(d'd(Z)da*da+d)d*Z,d+d,Z-Gd-d.d.ej.Z/Gd/d0d0ej0Z1e2edS)2z"Brian Quinlan (brian@sweetapp.com)N)_base)Full)wait)Queue)partialFc@s,eZdZddZddZddZddZd S) _ThreadWakeupcCstjdd\|_|_dS)NF)Zduplex)mpZPipe_reader_writer)selfr =/opt/alt/python37/lib/python3.7/concurrent/futures/process.py__init__Rsz_ThreadWakeup.__init__cCs|j|jdS)N)r closer )r r r r rUs z_ThreadWakeup.closecCs|jddS)N)r Z send_bytes)r r r r wakeupYsz_ThreadWakeup.wakeupcCsx|jr|jqWdS)N)r ZpollZ recv_bytes)r r r r clear\s z_ThreadWakeup.clearN)__name__ __module__ __qualname__rrrrr r r r rQsrcCsHdatt}x|D]\}}|qWx|D]\}}|q0WdS)NT)_global_shutdownlist_threads_wakeupsitemsrjoin)r_ thread_wakeuptr r r _python_exitas   r=c@seZdZddZddZdS)_RemoteTracebackcCs ||_dS)N)tb)r r"r r r rzsz_RemoteTraceback.__init__cCs|jS)N)r")r r r r __str__|sz_RemoteTraceback.__str__N)rrrrr#r r r r r!ysr!c@seZdZddZddZdS)_ExceptionWithTracebackcCs0tt|||}d|}||_d||_dS)Nz """ %s""") tracebackformat_exceptiontyperexcr")r r)r"r r r rs z _ExceptionWithTraceback.__init__cCst|j|jffS)N) _rebuild_excr)r")r r r r __reduce__sz"_ExceptionWithTraceback.__reduce__N)rrrrr+r r r r r$sr$cCst||_|S)N)r! __cause__)r)r"r r r r*s r*c@seZdZddZdS) _WorkItemcCs||_||_||_||_dS)N)futurefnargskwargs)r r.r/r0r1r r r rsz_WorkItem.__init__N)rrrrr r r r r-sr-c@seZdZdddZdS) _ResultItemNcCs||_||_||_dS)N)work_id exceptionresult)r r3r4r5r r r rsz_ResultItem.__init__)NN)rrrrr r r r r2sr2c@seZdZddZdS) _CallItemcCs||_||_||_||_dS)N)r3r/r0r1)r r3r/r0r1r r r rsz_CallItem.__init__N)rrrrr r r r r6sr6cs*eZdZdfdd ZfddZZS) _SafeQueuercs||_tj||ddS)N)ctx)pending_work_itemssuperr)r max_sizer8r9) __class__r r rsz_SafeQueue.__init__cslt|trZtt|||j}tdd||_ |j |j d}|dk rh|j |nt||dS)Nz """ {}"""r%) isinstancer6r&r'r( __traceback__r!formatrr,r9popr3r. set_exceptionr:_on_queue_feeder_error)r eobjr" work_item)r<r r rBs z!_SafeQueue._on_queue_feeder_error)r)rrrrrB __classcell__r r )r<r r7sr7cgs0t|}x"tt||}|s"dS|Vq WdS)N)ziptuple itertoolsislice) chunksize iterablesitchunkr r r _get_chunkss rOcsfdd|DS)Ncsg|] }|qSr r ).0r0)r/r r sz"_process_chunk..r )r/rNr )r/r _process_chunks rRc Cs^y|t|||dWn@tk rX}z"t||j}|t||dWdd}~XYnXdS)N)r5r4)r4)putr2 BaseExceptionr$r>) result_queuer3r5r4rCr)r r r _sendback_results   rVc Cs|dk r:y ||Wn$tk r8tjjddddSXx|jdd}|dkrb|tdSy|j|j |j }Wn>tk r}z t ||j }t ||j|dWdd}~XYnXt ||j|d~qrVr3) call_queuerU initializerinitargsZ call_itemrrCr)r r r _process_workers$    "r`cCsxxr|rdSy|jdd}Wntjk r4dSX||}|jrh|jt||j|j |j ddq||=qqWdS)NF)rXT) ZfullrYqueueZEmptyr.Zset_running_or_notify_cancelrSr6r/r0r1)r9Zwork_idsr\r3rEr r r _add_call_item_to_queues   rbc sFdfdd}fdd}|j} |j} | | g} x t||ddD} t| | } d}d}| | kry| }d}Wqtk r}ztt|||j }Wdd}~XYqXn| | krd}d}| |rt|dk rd _ d_ dt d }|dk r td d |d |_x$|D]\}}|j|~q*W| xD]}|qXW|dSt|tr|}|s|dSnL|dk r||jd}|dk r|jr|j|jn|j|j~~||r:y$dk rd_ |s |dSWntk r8YnXdq6WdS)NcstpdkpjS)N)r_shutdown_threadr )executorr r shutting_down?s z/_queue_management_worker..shutting_downc stddD}|}d}xn||kr|dkrxBt||D]2}yd|d7}Wq>tk rnPYq>Xq>WtddD}q WxD] }|qWdS)Ncss|]}|VqdS)N)is_alive)rPpr r r EszD_queue_management_worker..shutdown_worker..rrcss|]}|VqdS)N)rf)rPrgr r r rhQs)sumvaluesrangeZ put_nowaitrrr)Zn_children_aliveZn_children_to_stopZn_sentinels_sentirg)r\ processesr r shutdown_workerCs   z1_queue_management_worker..shutdown_workercSsg|] }|jqSr )sentinel)rPrgr r r rQhsz,_queue_management_worker..TFzKA child process terminated abruptly, the process pool is not usable anymorez^A process in the process pool was terminated abruptly while the future was running or pending.z ''' r%z''')r rbrjrZrecvrTr&r'r(r>r_brokenrcBrokenProcessPoolr!rr,rr.rAZ terminater=intr@r3r4Z set_resultr5r)Zexecutor_referencermr9Zwork_ids_queuer\rUrrernZ result_readerZ wakeup_readerZreadersZworker_sentinelsZreadycauseZ is_brokenZ result_itemrCZbper3rErgr )r\rdrmr _queue_management_worker!s  (       rtc Cshtrtrttdaytd}Wnttfk r:dSX|dkrHdS|dkrTdSd|attdS)NTSC_SEM_NSEMS_MAXz@system provides too few semaphores (%d available, 256 necessary))_system_limits_checked_system_limitedNotImplementedErrorrZsysconfAttributeError ValueError)Z nsems_maxr r r _check_system_limitssr~ccs.x(|D] }|x|r$|VqWqWdS)N)reverser@)iterableZelementr r r _chain_from_iterable_of_listss rc@s eZdZdS)rqN)rrrr r r r rqsrqcsheZdZdddZddZddZd d Zejjj e_ dd d fd d Z dddZ ejj j e _ Z S)ProcessPoolExecutorNr cCst|dkr6tpd|_tjdkrntt|j|_n8|dkrHtdn tjdkrh|tkrhtdt||_|dkr~t }||_ |dk rt |st d||_||_d|_i|_d|_t|_d|_d|_i|_|jt}t||j |jd|_d |j_||_t |_!t"|_#dS) Nrwin32rz"max_workers must be greater than 0zmax_workers must be <= zinitializer must be a callableF)r;r8r9T)$r~rZ cpu_count _max_workerssysplatformmin_MAX_WINDOWS_WORKERSr}rZ get_context _mp_contextcallable TypeError _initializer _initargs_queue_management_thread _processesrc threadingZLock_shutdown_lockrp _queue_count_pending_work_itemsEXTRA_QUEUED_CALLSr7 _call_queueZ _ignore_epipeZ SimpleQueue _result_queuerar _work_idsr_queue_management_thread_wakeup)r max_workersZ mp_contextr]r^Z queue_sizer r r rsF           zProcessPoolExecutor.__init__c Csv|jdkrr|jfdd}|tjtt|||j|j |j |j |j |jfdd|_d|j_ |j|jt|j<dS)NcSstjd|dS)Nz?Executor collected: triggering callback for QueueManager wakeup)rutildebugr)rrr r r weakref_cbAs zFProcessPoolExecutor._start_queue_management_thread..weakref_cbZQueueManagerThread)targetr0nameT)rr_adjust_process_countrZThreadrtweakrefrefrrrrrZdaemonstartr)r rr r r _start_queue_management_thread<s     z2ProcessPoolExecutor._start_queue_management_threadcCsTxNtt|j|jD]8}|jjt|j|j|j |j fd}| ||j|j <qWdS)N)rr0) rklenrrrZProcessr`rrrrrpid)r rrgr r r rWs z)ProcessPoolExecutor._adjust_process_countc Ost|dkr|^}}}n>|s&tdn0d|krB|d}|^}}ntdt|d|j|jrnt|j|jr|tdtrtdt }t ||||}||j |j <|j|j |j d7_ |j||SQRXdS)NzEdescriptor 'submit' of 'ProcessPoolExecutor' object needs an argumentr/z6submit expected at least 1 positional argument, got %drz*cannot schedule new futures after shutdownz6cannot schedule new futures after interpreter shutdown)rrr@rrprqrc RuntimeErrorrrZFuturer-rrrrSrrr)r0r1r r/fwr r r submitbs0        zProcessPoolExecutor.submitr)timeoutrKcs:|dkrtdtjtt|t|d|i|d}t|S)Nrzchunksize must be >= 1.rK)r)r}r:maprrRrOr)r r/rrKrLresults)r<r r rs  zProcessPoolExecutor.mapTc Cs|j d|_WdQRX|jr6|j|r6|jd|_|jdk rd|j|r^|jd|_d|_ d|_ |jr|jd|_dS)NT) rrcrrrrrrZ join_threadrr)r rr r r shutdowns"      zProcessPoolExecutor.shutdown)NNNr )T) rrrrrrrrExecutor__doc__rrrFr r )r<r rs J !  r)NN)3 __author__atexitrZconcurrent.futuresrrarZmultiprocessingrZmultiprocessing.connectionrZmultiprocessing.queuesrrr functoolsrrIrr&WeakKeyDictionaryrrrrrr Exceptionr!r$r*objectr-r2r6r7rOrRrVr`rbrtrxryr~rZBrokenExecutorrqrrregisterr r r r .sT         (&! L