Fa{@sddddgZddlZddlZddlZddlZddlmZddlmZdd l m Z dd l m Z dd l m Z dd l m Z dd l mZddl mZddl mZddZejejedddddfDZedek rJddZxeD]Zejeeq0WGdddeZfiddZddZGdddeZd d!Zd"d#ZGd$d%d%eZGd&d'd'eZ d(e j!e j"fd)e j#e j$fiZ%Gd*ddeZ&Gd+d,d,e'Z(Gd-ddeZ)d.d/Z*id0d1Z+dddd2d3d4Z,Gd5d6d6eZ-Gd7d8d8eZ.d2d9d:Z/Gd;d<d<e)Z0Gd=d>d>e)Z1Gd?d@d@e1Z2GdAdBdBe)Z3GdCdDdDe)Z4GdEdFdFe)Z5GdGdHdHe)Z6e+dIdJdKdLdMdNdOdPdQdRdSdTdUdVdWdXdYdZd[d\fZ7Gd]d^d^e7Z8e+d_dKdLdMdNdRd`dadbdcdddXdddedfdfZ9e+dgdNdMdRfZ:e+dhdidjdkdldmdndodpdqdrdsf Z;djdtdpdtdrdtdldudmduie;_<Gdvdhdhe;Z=Gdwdde&Z>e>jdxej?e>jdyej?e>jdzej@e3e>jd{ejAe1e>jd|ejBe1e>jd}ejCe1e>jd~ejDe1e>jdejEe2e>jdejFe4e>jde jGe=e>jdee8e>jdeHe9e>jd8e.e6e>jd:e/e:e>jd6e-e5e>jdude0dde>jdtdddS) BaseManager SyncManager BaseProxyTokenN)time) format_exc) connection)context)pool)process) reduction)util) get_contextcCstj|j|jffS)N)arraytypecodetobytes)ar./usr/lib/python3.5/multiprocessing/managers.py reduce_array%srcCs(g|]}tti|qSr)typegetattr).0namerrr )s ritemskeysvaluescCstt|ffS)N)list)objrrrrebuild_as_list+sr!c@sLeZdZdZdZddZddZd d Zd d Zd S)rz4 Type to uniquely indentify a shared object typeidaddressidcCs!||||_|_|_dS)N)r"r#r$)selfr"r#r$rrr__init__:szToken.__init__cCs|j|j|jfS)N)r"r#r$)r%rrr __getstate__=szToken.__getstate__cCs|\|_|_|_dS)N)r"r#r$)r%staterrr __setstate__@szToken.__setstate__cCs#d|jj|j|j|jfS)Nz %s(typeid=%r, address=%r, id=%r)) __class____name__r"r#r$)r%rrr__repr__CszToken.__repr__N)r"r#r$) r+ __module__ __qualname____doc__ __slots__r&r'r)r,rrrrr4s    cCsN|j||||f|j\}}|dkr;|St||dS)zL Send a message to manager using connection `c` and return response z#RETURNN)sendrecvconvert_to_error)cr$ methodnameargskwdskindresultrrrdispatchKs  r:cCs~|dkr|S|dkr>t|tks4tt|S|dkrpt|tksbttd|StdSdS)Nz#ERRORz #TRACEBACKz#UNSERIALIZABLEzUnserializable message: %s zUnrecognized message type)rstrAssertionError RemoteError ValueError)r8r9rrrr3Us    r3c@seZdZddZdS)r=cCs)ddddt|jdddS)N -Kr)r;r6)r%rrr__str__bszRemoteError.__str__N)r+r-r.rBrrrrr=as r=cCsIg}x<t|D].}t||}t|r|j|qW|S)z4 Return a list of names of methods of `obj` )dirrcallableappend)r temprfuncrrr all_methodsis  rHcCsddt|DS)zP Return a list of names of methods of `obj` which do not start with '_' cSs&g|]}|ddkr|qS)r_r)rrrrrrxs z"public_methods..)rH)r rrrpublic_methodstsrJc @seZdZdZdddddddd d g Zd d Zd dZddZddZddZ ddZ ddZ ddZ de de de iZ ddZd d!Zd"d#Zd$d%Zd&d'Zd(d)Zd*d+Zd,d-Zd.d/Zd0S)1ServerzM Server class which runs in a process controlled by a manager object shutdowncreateaccept_connection get_methods debug_infonumber_of_objectsdummyincrefdecrefcCst|tst||_tj||_t|\}}|d|dd|_|jj |_ ddffi|_ i|_ t j |_dS)Nr#Zbacklog0) isinstancebytesr<registryr AuthenticationStringauthkeylistener_clientlistenerr# id_to_objid_to_refcount threadingRLockmutex)r%rYr#r[ serializerListenerClientrrrr&s  zServer.__init__cCstj|_|tj_zqtjd|j}d|_|j y*x#|jj sq|jj dqOWWnt t fk rYnXWdtjtjkrtjdtjt_tjt_tjdXdS)z( Run the server forever targetTrNzresetting stdout, stderrr)r`Event stop_eventr current_process_manager_serverThreadaccepterdaemonstartis_setwaitKeyboardInterrupt SystemExitsysstdout __stdout__rdebug __stderr__stderrexit)r%rlrrr serve_forevers       zServer.serve_foreverc Csgx`y|jj}Wntk r-wYnXtjd|jd|f}d|_|jqWdS)Nrfr6T)r]ZacceptOSErrorr`rkhandle_requestrmrn)r%r4trrrrls  zServer.accepterc 'Csd}}}yvtj||jtj||j|j}|\}}}}||jksttd|t||}Wn!tk rdt f} YnGXy||||}Wn!tk rdt f} Yn Xd|f} y|j | Wntk r} zdy|j dt fWntk rBYnXt j d| t j d|t j d| WYdd} ~ XnX|j dS)z) Handle a new connection Nz%r unrecognizedz #TRACEBACKz#RETURNzFailure to send message: %rz ... request was %rz ... exception was %r)r Zdeliver_challenger[Zanswer_challenger2publicr<r Exceptionrr1rinfoclose) r%r4funcnamer9requestignorer6r7rGmsgerrrr|s4     #zServer.handle_requestc*Cs%tjdtjj|j}|j}|j}x|jj s y,d}}|}|\}}} } ||\}} } || krt d|t || ft ||} y| | | }Wn1t k r}zd|f}WYdd}~XnpX| o| j|d}|re|j|||\}}t||j|}d||ff}n d|f}Wnt k r|dkrdtf}n\y8|j|}|||||| | }d|f}Wn!t k rdtf}YnXYnXtk r7tjdtjjtjd Yn!t k rWdtf}YnXyMy||Wn8t k r}z|d tfWYdd}~XnXWq7t k r}zUtjd tjjtjd |tjd ||jtjdWYdd}~Xq7Xq7WdS)zQ Handle requests from the proxies in a particular process/thread z$starting server thread to service %rNz+method %r of %r object is not in exposed=%rz#ERRORz#PROXYz#RETURNz #TRACEBACKz$got EOF -- exiting thread serving %rrz#UNSERIALIZABLEzexception in thread serving %rz ... message was %rz ... exception was %rr)rrvr`current_threadrr2r1r^rhroAttributeErrorrrrgetrMrr#rfallback_mappingEOFErrorrsryrr)r%connr2r1r^r5r ridentr6r7exposedZ gettypeidZfunctionZresrrr"ZridentZrexposedtokenZ fallback_funcr9rrr serve_clientsl              *  zServer.serve_clientcCs|S)Nr)r%rrr rrrfallback_getvalueszServer.fallback_getvaluecCs t|S)N)r;)r%rrr rrr fallback_str"szServer.fallback_strcCs t|S)N)repr)r%rrr rrr fallback_repr%szServer.fallback_reprrBr,z #GETVALUEcCsdS)Nr)r%r4rrrrR.sz Server.dummycCs|jg}t|jj}|jxV|D]N}|dkr6|jd||j|t|j|dddfq6Wdj|SWdQRXdS)zO Return some info --- useful to spot problems with refcounting rVz %s: refcount=%s %srNrAr?) rbrr^rsortrEr_r;join)r%r4r9rrrrrrP1s      *zServer.debug_infocCst|jdS)z* Number of shared objects r)lenr^)r%r4rrrrQ@szServer.number_of_objectscCsXzCytjd|jdWnddl}|jYnXWd|jjXdS)z' Shutdown this process z!manager received shutdown message#RETURNNr)rN)rrvr1 traceback print_excrhset)r%r4rrrrrLFs  zServer.shutdownc OsB|j2|j|\}}}}|dkr[t|dkrH| sNt|d} n|||} |dkrt| }|dk rt|tkstt|t|}dt| } t j d|| | t ||f|j | <| |j krd|j | <|j|| | t|fSWdQRXdS)z> Create a new shared object and return its id Nrrz%xz&%r callable returned object with id %r)rbrYrr<rJrdictrr$rrvrr^r_rStuple) r%r4r"r6r7rDrmethod_to_typeid proxytyper rrrrrMSs$       z Server.createcCst|j|jdS)zL Return the methods of the shared object indicated by token r)rr^r$)r%r4rrrrrOvszServer.get_methodscCs-|tj_|jd|j|dS)z= Spawn a new thread to serve this connection #RETURNN)rN)r`rrr1r)r%r4rrrrrN|s zServer.accept_connectionc Cs&|j|j|d7 Connect manager object to the server process r[NrR) r\rrrr:rrrr)r%rdrerrrrconnectszBaseManager.connectc Csj|jjtjkst|dk r@t| r@tdtjdd\}}|j j dt |j d|j |j|j|j|||f|_djdd |jjD}t |jd ||j_|jj|j|j|_|jtj|j_tj|t |jd|j|j|j|j|jfd d |_dS) z@ Spawn a server process for this manager object Nzinitializer must be a callableZduplexFrfr6:css|]}t|VqdS)N)r;)rirrr sz$BaseManager.start..r@ exitpriorityr)rrrrr<rD TypeErrorr ZPiperZProcessr _run_serverrrrr_processrZ _identityr+rrnrr2rrFinalize_finalize_managerrrL)r% initializerinitargsreaderwriterrrrrrns(  "   zBaseManager.startc Csi|dk r|||j||||}|j|j|jtjd|j|jdS)z@ Create a server, report its address and run it Nzmanager serving at %r)_Serverr1r#rrrrz) clsrYr#r[rcrrrserverrrrrs   zBaseManager._run_serverc Os|jjtjks!td|j|jd|j}z)t|dd|f||\}}Wd|j Xt ||j||fS)zP Create a new shared object; return the token and exposed tuple zserver not yet startedr[NrM) rrrrr<rrrr:rr)r%r"r6r7rr$rrrr_creates !) zBaseManager._createcCs;|jdk r7|jj||jjs7d|_dS)zC Join the manager process (if it has been spawned) N)rris_alive)r%timeoutrrrrszBaseManager.joinc CsA|j|jd|j}zt|ddSWd|jXdS)zS Return some info about the servers shared objects and connections r[NrP)rrrr:r)r%rrrr _debug_infoszBaseManager._debug_infoc CsA|j|jd|j}zt|ddSWd|jXdS)z5 Return the number of shared objects r[NrQ)rrrr:r)r%rrrr_number_of_objects"szBaseManager._number_of_objectscCs>|jjtjkr|j|jjtjks:t|S)N)rrrrrnrr<)r%rrr __enter__,s zBaseManager.__enter__cCs|jdS)N)rL)r%exc_typeexc_valexc_tbrrr__exit__2szBaseManager.__exit__cCs|jrtjdy8||d|}zt|ddWd|jXWntk reYnX|jdd|jrtjdt|drtjd |j|jdd |jrtjd t j |_ yt j |=Wntk r YnXdS) zQ Shutdown the manager process; will be registered as a finalizer z#sending shutdown message to managerr[NrLrg?zmanager still alive terminatez'trying to `terminate()` manager processg?z#manager still alive after terminate)rrrr:rrrhasattrrrrrr_address_to_localKeyError)r r#r[r(rrrrrr5s.           zBaseManager._finalize_managercCs|jS)N)r)r%rrrUszBaseManager.Tc s!d|jkr!|jj|_dkr3t|pHtdd}|p`tdd}|rxdt|jD]P\}}t|tkst d|t|tks|t d|q|W|||f|j<|rfdd} | _ t || dS)z9 Register a typeid with the manager type rN _exposed__method_to_typeid_z%r is not a stringc stjd|j||\}}||jd|d|jd|}|j|jd|j}t|dd|jf|S)Nz)requesting creation of a shared %r objectmanagerr[rrT) rrvrrrrr#r:r$)r%r6r7rZexpproxyr)rr"rrrFrsz"BaseManager.register..temp) __dict__rcopy AutoProxyrrrrr;r<r+setattr) rr"rDrrr create_methodkeyrrFr)rr"rregisterWs  "& zBaseManager.register)r+r-r.r/rrKrr&rrrn classmethodrrrrrrr staticmethodrpropertyr#rrrrrrs*    $     c@s(eZdZddZddZdS)ProcessLocalSetcCstj|dddS)NcSs |jS)N)clear)r rrrrsz*ProcessLocalSet.__init__..)rregister_after_fork)r%rrrr&szProcessLocalSet.__init__cCst|ffS)N)r)r%rrr __reduce__szProcessLocalSet.__reduce__N)r+r-r.r&rrrrrrs  rc@seZdZdZiZejZddddddZddZ fidd Z d d Z d d Z e ddZddZddZddZddZddZdS)rz. A base for proxies of shared objects NTc Cs*tjNtjj|jd}|dkrStjtf}|tj|j)rr+rr"r$)r%rrrr,%szBaseProxy.__repr__c CsBy|jdSWn*tk r=t|dddSYnXdS)zV Return representation of the referent (or a fall-back if that fails) r,Nrz; '__str__()' failed>r)rrr)r%rrrrB)s zBaseProxy.__str__)r+r-r.r/rrZForkAwareThreadLockrr&rrrrrrrrrr,rBrrrrrs  "      cCsttjdd}|rB|j|jkrB|j|jdS|jddojttjdd }|||d||SdS)z Function used for unpickling proxy objects. If possible the shared object is returned, or otherwise a proxy for it. rjNrrSTZ _inheritingF)rr rir#r^r$pop)rGrrcr7rrSrrrr6s rc Cst|}y|||fSWntk r2YnXi}x%|D]}td||f|q@Wt|tf|}||_||||f<|S)zB Return a proxy type whose methods are given by `exposed` zLdef %s(self, *args, **kwds): return self._callmethod(%r, args, kwds))rrexecrrr)rr_cacheZdicmeth ProxyTyperrr MakeProxyTypeKs    rTc Cst|d}|dkrY||jd|}zt|dd|f}Wd|jX|dkrz|dk rz|j}|dkrtjj}td|j |}|||d|d|d|} d| _ | S) z* Return an auto-proxy for `token` rNr[rOz AutoProxy[%s]rrST) r\r#r:rrr rir[rr"r) rrcrr[rrSrrrrrrrras      rc@s(eZdZddZddZdS) NamespacecKs|jj|dS)N)rupdate)r%r7rrrr&szNamespace.__init__cCst|jj}g}x:|D]2\}}|jds"|jd||fq"W|jd|jjdj|fS)NrIz%s=%rz%s(%s)z, ) rrr startswithrErr*r+r)r%rrFrrrrrr,s zNamespace.__repr__N)r+r-r.r&r,rrrrr~s  rc@sReZdZdddZddZddZdd ZeeeZd S) ValueTcCs||_||_dS)N) _typecode_value)r%rrlockrrrr&s zValue.__init__cCs|jS)N)r)r%rrrrsz Value.getcCs ||_dS)N)r)r%rrrrrsz Value.setcCs dt|j|j|jfS)Nz %s(%r, %r))rr+rr)r%rrrr,szValue.__repr__N) r+r-r.r&rrr,rrrrrrrs    rcCstj||S)N)r)rZsequencerrrrArraysrc@sReZdZdZddZddZd d Zd d Zd dZdS) IteratorProxy__next__r1throwrcCs|S)Nr)r%rrr__iter__szIteratorProxy.__iter__cGs|jd|S)Nr)r)r%r6rrrrszIteratorProxy.__next__cGs|jd|S)Nr1)r)r%r6rrrr1szIteratorProxy.sendcGs|jd|S)Nr)r)r%r6rrrrszIteratorProxy.throwcGs|jd|S)Nr)r)r%r6rrrrszIteratorProxy.closeN)rr1rr) r+r-r.rrrr1rrrrrrrs     rc@sLeZdZd ZddddZddZd d Zd d ZdS) AcquirerProxyacquirereleaseTNcCs1|dkr|fn ||f}|jd|S)Nr )r)r%Zblockingrr6rrrr s!zAcquirerProxy.acquirecCs |jdS)Nr )r)r%rrrr szAcquirerProxy.releasecCs |jdS)Nr )r)r%rrrrszAcquirerProxy.__enter__cCs |jdS)Nr )r)r%rrrrrrrszAcquirerProxy.__exit__)r r )r+r-r.rr r rrrrrrr s   r c@sLeZdZdZdddZd d Zd d Zdd dZdS)ConditionProxyr r rpnotify notify_allNcCs|jd|fS)Nrp)r)r%rrrrrpszConditionProxy.waitcCs |jdS)Nr )r)r%rrrr szConditionProxy.notifycCs |jdS)Nr)r)r%rrrrszConditionProxy.notify_allcCs|}|r|S|dk r/t|}n d}d}xF|s|dk rj|t}|dkrjP|j||}q>W|S)Nr)_timerp)r%Z predicaterr9ZendtimeZwaittimerrrwait_fors        zConditionProxy.wait_for)r r rpr r)r+r-r.rrpr rrrrrrr s   r c@sIeZdZdZddZddZd d Zd d d Zd S) EventProxyrorrrpcCs |jdS)Nro)r)r%rrrroszEventProxy.is_setcCs |jdS)Nr)r)r%rrrrszEventProxy.setcCs |jdS)Nr)r)r%rrrrszEventProxy.clearNcCs|jd|fS)Nrp)r)r%rrrrrpszEventProxy.wait)rorrrp)r+r-r.rrorrrprrrrrs    rc@sseZdZdZdddZdd Zd d Zed d ZeddZ eddZ dS) BarrierProxy__getattribute__rpabortresetNcCs|jd|fS)Nrp)r)r%rrrrrpszBarrierProxy.waitcCs |jdS)Nr)r)r%rrrrszBarrierProxy.abortcCs |jdS)Nr)r)r%rrrrszBarrierProxy.resetcCs|jddS)Nrparties)r)r)r%rrrrszBarrierProxy.partiescCs|jddS)Nr n_waiting)r)r)r%rrrrszBarrierProxy.n_waitingcCs|jddS)Nrbroken)r)r)r%rrrrszBarrierProxy.broken)rrprr) r+r-r.rrprrrrrrrrrrrs   rc@s:eZdZd ZddZddZdd Zd S) NamespaceProxyr __setattr__ __delattr__cCsB|ddkr tj||Stj|d}|d|fS)NrrIrr)objectr)r%r callmethodrrr __getattr__szNamespaceProxy.__getattr__cCsH|ddkr#tj|||Stj|d}|d||fS)NrrIrr)rrr)r%rrrrrrrszNamespaceProxy.__setattr__cCsB|ddkr tj||Stj|d}|d|fS)NrrIrr)rrr)r%rrrrrrszNamespaceProxy.__delattr__N)rrr)r+r-r.rrrrrrrrrs   rc@s=eZdZdZddZddZeeeZdS) ValueProxyrrcCs |jdS)Nr)r)r%rrrrszValueProxy.getcCs|jd|fS)Nr)r)r%rrrrr szValueProxy.setN)rr)r+r-r.rrrrrrrrrrs   r BaseListProxy__add__ __contains__ __delitem__ __getitem____len____mul__ __reversed____rmul__ __setitem__rEcountextendindexinsertrremovereverser__imul__c@s(eZdZddZddZdS) ListProxycCs|jd|f|S)Nr+)r)r%rrrr__iadd__szListProxy.__iadd__cCs|jd|f|S)Nr0)r)r%rrrrr0szListProxy.__imul__N)r+r-r.r2r0rrrrr1s  r1 DictProxyrrrZhas_keypopitem setdefaultr ArrayProxy PoolProxyZapplyZ apply_asyncrZimapZimap_unorderedrmapZ map_asyncstarmapZ starmap_asyncrZ AsyncResultIteratorc@s(eZdZddZddZdS)r7cCs|S)Nr)r%rrrr5szPoolProxy.__enter__cCs|jdS)N)r)r%rrrrrrr7szPoolProxy.__exit__N)r+r-r.rrrrrrr74s  c@seZdZdZdS)ra( Subclass of `BaseManager` which supports a number of shared object types. The types registered are those intended for the synchronization of threads, plus `dict`, `list` and `Namespace`. The `multiprocessing.Manager()` function creates started instances of this class. N)r+r-r.r/rrrrr>s QueueZ JoinableQueuergLockra SemaphoreBoundedSemaphore ConditionBarrierPoolrrrrF)I__all__rsr`rZqueuerrrrr r r r r rrrrZ view_typesrr!Z view_typerrr:r3rr=rHrJrKrrdreZ XmlListenerZ XmlClientr\rrrrrrrrrrrr r rrrrr r1r3r6Z BasePoolProxyrr7rr;rgr<rar=r>r?r@rArrrrr s