^cl@dZddlZddlZddlZddlZddlZddlmZddlm Z ddl m Z ddl mZm Z mZddlmZejZejZejZdaGd d ZGd d ZGd deejZGddejZdZdZGddejZdZdZddZdZ ddZ!e"dkr]ddl#Z# e$ej%dZ&ej%dZ'e!e'e&dS#e(e)f$re*dej%dzYdSwxYwdS) aKAn HTTP handler for urllib2 that supports HTTP 1.1 and keepalive. >>> import urllib2 >>> from keepalive import HTTPHandler >>> keepalive_handler = HTTPHandler() >>> opener = urlreq.buildopener(keepalive_handler) >>> urlreq.installopener(opener) >>> >>> fo = urlreq.urlopen('http://www.python.org') If a connection to a given host is requested, and all of the existing connections are still in use, another connection will be opened. If the handler tries to use an existing connection but it fails in some way, it will be closed and removed from the pool. To remove the handler, simply re-run build_opener with no arguments, and install that opener. You can explicitly close connections by using the close_connection() method of the returned file-like object (described below) or you can use the handler methods: close_connection(host) close_all() open_connections() NOTE: using the close_connection and close_all methods of the handler should be done with care when using multiple threads. * there is nothing that prevents another thread from creating new connections immediately after connections are closed * no checks are done to prevent in-use connections from being closed >>> keepalive_handler.close_all() EXTRA ATTRIBUTES AND METHODS Upon a status of 200, the object returned has a few additional attributes and methods, which should not be used if you want to remain consistent with the normal urllib2-returned objects: close_connection() - close the connection to the host readlines() - you know, readlines() status - the return status (i.e. 404) reason - english translation of status (i.e. 'File not found') If you want the best of both worlds, use this inside an AttributeError-catching try: >>> try: status = fo.status >>> except AttributeError: status = None Unfortunately, these are ONLY there if status == 200, so it's not easy to distinguish between non-200 responses. The reason is that urllib2 tries to do clever things with error codes 301, 302, 401, and 407, and it wraps the object upon return. N)_)getattr)hex)pycompat urllibcompatutil)procutilc8eZdZdZdZdZdZdZdZd dZ dS) ConnectionManagerzT The connection manager must be able to: * keep track of all existing ctj|_tjt |_i|_i|_dSN) threadingLock_lock collections defaultdictlist_hostmap_connmap _readymapselfs 5/usr/lib/python3/dist-packages/mercurial/keepalive.py__init__zConnectionManager.__init__ss5^%% #/55  c|j |j||||j|<||j|<|jdS#|jwxYwr)racquirerappendrrrelease)rhost connectionreadys raddzConnectionManager.addys  ! M$  & &z 2 2 2(,DM* %).DN: & J   DJ   s 4A**Bcp|j |j|}|j|=|j|=|j|||j|s|j|=n#t $rYnwxYw|jdS#|jwxYwr)rrrrrremoveKeyErrorr )rr"r!s rr&zConnectionManager.removes  ! ,}Z0M*-N:. d#**:666}T*, d+     J   DJ   s) A/AB/ A<9B;A<<BB5c> ||j|<dS#t$rYdSwxYwr)rr')rr"r#s r set_readyzConnectionManager.set_readys; ).DN: & & &    DD s  cd}|j |j|D]}|j|rd|j|<|}n|jn#|jwxYw|S)NF)rrrrr )rr!conncs rget_ready_connz ConnectionManager.get_ready_conns  !]4(  >!$(-DN1%DE J   DJ    s +A""A=Nc|rt|j|Std|jDS)Nc4i|]\}}|t|S)r).0hconnss r z-ConnectionManager.get_all..s$HHHJQDKKHHHr)rrdictitems)rr!s rget_allzConnectionManager.get_allsP   d+,, ,HH$-2E2E2G2GHHH rr) __name__ __module__ __qualname____doc__rr$r&r)r-r7r0rrr r ms}  !!!!!!    rr cNeZdZd dZdZdZdZdZddZd Z d Z d Z d Z dS)KeepAliveHandlerNcVt|_||_d|_d|_dSNr)r _cm_timeout requestscountsentbytescount)rtimeouts rrzKeepAliveHandler.__init__s,$&& rcld|jDS)ztreturn a list of connected hosts and the number of connections to each. [('foo.com:80', 2), ('bar.org', 1)]c6g|]\}}|t|fSr0)len)r1r!lis r z5KeepAliveHandler.open_connections..s&MMMJT2s2wwMMMr)r@r7r6rs ropen_connectionsz!KeepAliveHandler.open_connectionss3NM$(2B2B2D2D2J2J2L2LMMMMrc|j|D]0}|j||1dS)zclose connection(s) to host is the host:port spec, as in 'www.cnn.com:8080' as passed in. no error occurs if there is no connection to that host.N)r@r7r&close)rr!r2s rclose_connectionz!KeepAliveHandler.close_connectionsP!!$''  A HOOA    GGIIII  rc|jD]8\}}|D]0}|j||19dS)zclose all open connectionsN)r@r7r6r&rL)rr!r3r2s r close_allzKeepAliveHandler.close_allsp8++--3355  KD%  """    rc<|j|ddS)zetells us that this request is now closed and that the connection is ready for another requestTN)r@r))rrequestr!r"s r_request_closedz KeepAliveHandler._request_closeds" :t,,,,,rrcf|r||j|dSr)rLr@r&)rr!r"rLs r_remove_connectionz#KeepAliveHandler._remove_connections6          #####rc8|t|Sr)do_openHTTPConnection)rreqs r http_openzKeepAliveHandler.http_opens||NC000rctj|}|std |j|}|rd||||}|rn||j||j|}|d|||j }tr)t d|t||j ||d||||}n#t j$rF}tt%dt'j|jzd}~wt,jt jf$r}t|d}~wwxYwt3|ddr|j|tr&t d|j|j||_||_||_||_ |j|_!|j"|_#|j|_"|S) Ns no host given)rDs"creating new connection to %s (%d)Fsbad HTTP status line: %s will_closeTsSTATUS: %s, %s)$rgethosturlerrurlerrorr@r-_reuse_connectionrLr&rADEBUGinfoidr$_start_transaction getresponsehttplib BadStatusLinerrsysbyteslinesocketerror HTTPExceptionrstatusreason_handler_host get_full_url_url _connectioncodemsgheaders)r http_classrXr!r2rerrs rrVzKeepAliveHandler.do_opensW#C(( 4//"233 3 '''--A $**1c488 """H++D11 $JtT];;;JJ=tRUU T1e,,,''3///MMOO$   //-..1B381L1LL  g34 ' ' '//#&& & ' 1lD ) )  HOOA     > JJ(!(AH = = = !!## E s%DD;;G AF  G)GGcB ||||}n#tjtjf$rd}Ynct r)t d|t||j || xYw| |j dkr3t r)t d|t|d}n0t r)t d|t||S)aGstart the transaction with a re-used connection return a response object (r) upon success or None on failure. This DOES not close or remove bad connections in cases where it returns. However, if an unexpected exception occurs, it will close and remove the connection before re-raising. Ns4unexpected exception - closing connection to %s (%d) s&failed to re-use connection to %s (%d)sre-using connection to %s (%d)) rcrdrirjrerkr`rbr@r&rLversionra)rr2rXr!rws rr_z"KeepAliveHandler._reuse_connections!   # #As + + + AA g34   AAA   KqEE HOOA    GGIII  9 Q   =tRUUAA K z6KeepAliveHandler._start_transaction..Ls2KK41aAKKKKKKr)r!zaccept-encodingrskip_-rz content-typez Content-typez!application/x-www-form-urlencodedzcontent-lengthzContent-lengthz%d)rr sortdictparent addheadersupdatesortedrur6unredirected_hdrsreplacerhasdatagetdata putrequest get_method getselector putheaderrGrirjr]r^ endheaderssendrBrCAttributeError) rr2rX oldbytescountru skipheadersrdatarxkrs rrcz#KeepAliveHandler._start_transactionCs#3Q77 - 677vck//1122333vc399;;<<===-KK7==??KKKKK , ? ?AG||=> GaiiS&9&99: '#C(( #+C00 NN$$ ,S11" "00KK&(K$722KK 0$T2BCCC NN$$ ,S11" | ' ' '//#&& & 'MMOO  DAq KK1        $ $  FF4LLL a wq*:A>>NN  K % % * % % K & &+Q//-?  & & & &    DD s+:CGG<G77G<|j}g} ||j}|d}|dkr |d|} t |d}nO#t $rB|td |wxYw|dkrn|)| | |n||krG| | |||z |_d |S||krY| | || dd|_d |S| | |||z}| dd} |j}|sn|dkrn$|d |S)NT;rrs ) chunk_leftrreadlinefindint ValueErrorrLreIncompleteReadjoinr _safe_read)rrrpartsrhis r _read_chunkedzHTTPResponse._read_chunkeds_  !w''))IIdOO668DB!$T2JJ!BBBJJLLL!00%AAA B ??{ T__Z889999z!! T__S11222",s"2xx& "" T__S11222""""&xx& T__Z88999z! OOA   JA H 7##%%D w  xxs AA B)c|jd}|dkr*|dz }|jd|}|j|d|_|S|j |jg}ng}d}|j} ||}|sn|xjt |z c_|jxjt |z c_ |jj xjt |z c_n#t$rYnwxYw| ||d}|dkrn|dkrd|_d |S|dz }|d|d|_|dd||d<d |S)N rrTr) rr _raw_readintorrrrGrrrnrrrr)rrrhchunksreadsizenews rrzHTTPResponse.readlines JOOE " " 66 FA:bqb>DABBDJK   %j\FFF > ..**C   # #s3xx / # #   / /3s88 ; / /  $773s88C777!     MM#   AAvv ( 77DJ88F## # QBZ^ BZ^r xxs?'C'' C43C4cd}g} |}|sn1|||t|z }|r||krnH|Sr?)rrrG)rsizehinttotalrrhs r readlineszHTTPResponse.readlinesAsl ==??D  KK    SYY E EX--  rc|jI|t|}|sdS||dt|<t|St|}t|j}||kr*|jd||d|<|j|d|_|St |}||||}|xj|z c_|jxj|z c_ |jxj|z c_n#t$rYnwxYw|j|d|<|t|jz }d|_|S)Nrr) rrrGr memoryviewrrrrnr)rdestresrhavemvgots rrzHTTPResponse.readintoNs^   %))CII&&C q!$DSXX s88OD 4: 5== Jvv.D5MEFF+DJL     DJ00 3& ++s2++  M , , 3 , , ,    D zQtV  s4:  s6D DD)rrNrr)r8r9r:rrerrrrrrLrrMrarrrrrr0rrrr~s&     $)IGG0*dCCM@444l, , , \    rrct|dddS|j(|jr|n tj|jdkrtdt| d}t|dd}|o|jdkrtd||}|rF|j ||xj t|z c_ ||}|DdSdS|j ||xj t|z c_ dS#t$rU|j tjkr#d|_||_d}nd }||rYdSwxYw) z`Send `str' to the server. Shamelessly ripped off from httplib to patch a bad behavior. _broken_pipe_respNrssend:i rssending a read()ableFT)rr auto_openconnectre NotConnectedrprintreprsendallrCrGBrokenPipeError_HTTPConnection__state _CS_REQ_SENTrrdrL)rstr blocksizerrreraises rsafesendrkst($//; y > ' LLNNNN& &  hS """ sFD))  ""-...4 ??D ' !!$'''##s4yy0##tI ' ' ' ' ' I  c " " "   3s88 +          &'*> > >%)D "%)%5%5%7%7D "GGG       s,BD)07D))AFFc4fd}jj|_|S)z9Wraps getresponse in cls with a broken-pipe sane version.cXt|dd}||S|S)Nr)rrd)rrwclss rsafegetresponsez(wrapgetresponse..safegetresponses3 D-t 4 4 =Ht$$$r)rdr;)rrs` rwrapgetresponsers1%%%%%"o5O rcLeZdZeZeZeej Z dZ fdZ xZ S)rWcVtjj|g|Ri|d|_d|_dSr?)rerWrrCr)rargskwargss rrzHTTPConnection.__init__s<'>t>>>v>>>"#rctt|}d}|j}|r) d|z}n#t $rYnwxYwd|||j|jfzS)Nz (unconnected)z%s:%dz<%s: %s <--> %s:%d>)superrW__repr__r getsocknameOSErrorr!port)rbaselocalr __class__s rrzHTTPConnection.__repr__s^T**3355 I   !--//1    $eTY 'JJJsA AA)r8r9r:rresponse_classrrrrerWrdrr __classcell__)rs@rrWrWsn "N D!/'"899K$$$ K K K K K K K K KrrWc"tj}d}t}t|t|}|}|||}t|dt| fztt}t|t|}|}|||}t|dt| fzt|}d} | }|r||z}nn|||}t|dt| fzdS)Ns%25s: %ss normal urllibskeepalive readrTskeepalive readline) hashlibmd5urlreq buildopener installopenerurlopenrrLrrdigestrr)urlrformatopenerfofoomfs r continuityr s +C F   ! !F      B ''))CHHJJJ CA &$c!((**oo6 6777   . .F      B ''))CHHJJJ CA &%s188::7 7888   B C KKMM  'CC   HHJJJ CA &)3qxxzz??; ;<<<<.FakeLoggerc*t||zdSr)r)rrtrs rdebugz&test_timeout..FakeLogger.debug%s #*     rN)r8r9r:rrawarningrjr0rr FakeLoggerr$s.   "'&&wrr!s- fetching the file to establish a connections; waiting %i seconds for the server to close the connectionrs %2ir s! fetching the file a second times data are identicals ERROR: DATA DIFFER) r`rrrrrLr r r flushrrstderr)rdbbackupr!rdata1rdata2s r test_timeoutr) slH'''''''' JLLE :;;;   B GGIIEHHJJJ A H1 LMMM a%%j1n--- 1  Q a%%  O%    .///   B GGIIEHHJJJ ~~ %&&&& %&&& EEEr ctdt|tdtdt||tdtdt|dS)Ns>performing continuity test (making sure stuff isn't corrupted)rsperforming speed comparisons#performing dropped-connection check)rr rr))rrs rtestr,Fsq KLLLsOOO #JJJ ()))CLLL #JJJ 0111r__main__rs%s r)r*)+r;rrrisysri18nrrrnoderrr utilsr rer]rr`r r= httphandlerrrrrrWr rr r)r,r8rrargvrr IndexErrorrrr0rrr6s477x   ,   >>>>>>>>BIIIIIIIIX     "F$6   jjjjj7'jjjZ333l"KKKKKW+KKK@!=!=!=H555&.###L zKKK C  hqk S!   #333 #chqk12222223 s#C77 DD