a 'DgЊ@sddlZddlZddlZddlZddlZddlZddlZddlZddlm Z m Z m Z ddl m Z ddlmZddlmZddlmZddlmZddlmZmZmZmZmZmZmZdd lmZm Z m!Z!m"Z"ddl#Z#dd l#m$Z$dd l%m&Z&m'Z'm(Z(e)e*Z+d Z,d dZ-ddZ.e/dddZ0dBe1e2e3ddddZ4e1ddddZ5eddddZ6Gd d!d!Z7Gd"dde7Z8Gd#dde7Z9Gd$d%d%Z:Gd&d'd'e;Ze:d.d/d0Z?dDed1efe1e/ej@ee2ed2d3d4ZAdEed1efee1e2e/ed7d8d9ZBe2d:de+jCdddddd-d+d5f e2ee2eeeee2eeeeee2ge2fe1e>e2d; dd?d?ZEdFd@dAZFdS)GN)ThreadPoolExecutor TimeoutError as_completed) parsedate)partial) NOT_FOUND)count)create_default_context)AnyCallableIteratorListOptionalTupleUnion)quoteurlparseurlsplit urlunparse) exceptions) performanceutilversionREDACTEDcCs<tt|dd}|ds4|dr4|d|d<d|d<t|S)Nhttp)scheme)listrr)urlZ parsed_urlr!8/usr/lib/python3.9/site-packages/cloudinit/url_helper.py _cleanurl&s  r#cGs$dd}|}|D]}|||}q|S)NcSsNtt|}|d}|r*|ds*|d7}|tt|dd7}||d<t|S)Nr/z/:)safe)rrendswithrstrr)r add_onZ url_parsedpathr!r!r"combine_single1s z#combine_url..combine_singler!)baseZadd_onsr*r r(r!r!r" combine_url0s   r,returnc CsVtjdtjdtjdtjdtdi}|t|}|sRt|t rD|j }nt dd}|S)z4helper for read_ftps to map return codes to a number,iXiz9Unexpected exception type while connecting to ftp server.i) ftplibZ error_replyZ error_temp error_permZ error_protoEOFErrorgettype isinstanceOSErrorerrnoLOGwarning)excZftp_error_codescoder!r!r""ftp_get_return_code_from_exception@s r>@ FtpResponse)r timeoutkwargsr.c Ksrt|}|jstdtd|dt4}|jp2d}|jpZloginpasswordZprot_pr3r;r)Z retrbinarywriter@getvaluecloseZFTP) r rArBZ url_partsbufferrJrLZftp_tlser=ftpr!r!r" read_ftpsWs                  r\ FileResponse)r)r.c Ks|drtd|zt|}t||WStyb}zt|td|d|WYd}~n@d}~0t y}z t||j d|d|WYd}~n d}~00dS)zmread a binary file and return a FileResponse matches function signature with read_ftps and read_url dataz'Unable to post data to file resource %sNrC) r5r:r;rZload_binary_filer]FileNotFoundErrorrPrIOErrorr9)r)rBcontentsrZr!r!r" _read_files    &rb)r] UrlResponser@c Ks|}z t|}Wn2tyF}zt||d|WYd}~n d}~00|j}|dksf|rxd|dkrxt|jfi|S|dvrt|fi|S|dvrt|fi|St d|t|fi|SdS) a0Wrapper function around readurl to allow passing a file path as url. When url is not a local file path, passthrough any kwargs to readurl. In the case of parameter passthrough to readurl, default values for some parameters. See: call-signature of readurl in this module for param docs. )rDr Nfiler$r)r[rG)rhttpszAttempting unknown protocol %s) lstripr ValueErrorrPrrbr)r\readurlr:r;)r rBparsedrZrr!r!r"read_file_or_urls  $ rjc@s&eZdZd ddZddZddZdS) StringResponsecCs||_i|_||_||_dSN)r=rErar selfrar r=r!r!r"__init__szStringResponse.__init__cOs |jdkS)Nrlr=roargsrBr!r!r"okszStringResponse.okcCs |jdS)Nzutf-8)radecoderor!r!r"__str__szStringResponse.__str__N)rl)__name__ __module__ __qualname__rprtrwr!r!r!r"rks rkcs&eZdZdeedfdd ZZS)r]rlrar cstj|||ddS)Nrqsuperrprn __class__r!r"rpszFileResponse.__init__)rlrxryrzbytesr'rp __classcell__r!r!r~r"r]scs$eZdZeedfdd ZZS)r@r{cst||dSrmr|)rorar r~r!r"rpszFtpResponse.__init__rr!r!r~r"r@sc@seZdZejdddZeedddZee dddZ de dd d Z ed d Z eedddZddZdeee eedddZdS)rc)responsecCs ||_dSrm) _response)rorr!r!r"rp szUrlResponse.__init__r-cCs|jjdurdS|jjS)N)rcontentrvr!r!r"ra#s zUrlResponse.contentscCs|jjSrm)rr rvr!r!r"r )szUrlResponse.urlFcCs2d}|r d}d|jkr"|kr*nndSdSdS)Nr/r0rlTFrq)roZ redirects_okupperr!r!r"rt-s zUrlResponse.okcCs|jjSrm)rrErvr!r!r"rE6szUrlResponse.headerscCs|jjSrm)r status_codervr!r!r"r=:szUrlResponse.codecCs|jjSrm)rtextrvr!r!r"rw>szUrlResponse.__str__r) chunk_sizedecode_unicoder.ccs|j||EdHdS)atIterates over the response data. When stream=True is set on the request, this avoids reading the content at once into memory for large responses. :param chunk_size: Number of bytes it should read into memory. :param decode_unicode: If True, content will be decoded using the best available encoding based on the response. N)r iter_content)rorrr!r!r"rAs zUrlResponse.iter_contentN)F)rF)rxryrzrequestsZResponserppropertyrrar'r boolrtrEintr=rwrr rr!r!r!r"rcs   rcc@seZdZdddZdS)rPNcCs<t|t|||_||_||_|jdur2i|_||_dSrm)r`rpr'rDr=rEr )rorDr=rEr r!r!r"rpQs zUrlError.__init__)NNN)rxryrzrpr!r!r!r"rPPsrPcCsi}t|j}|dkr~|r~d|vr8|dr8|d|d<nd|d<d|vrfd|vrf|d|dg|d<nd|vr~t|d|d<|S)NreZca_certsZverifyTZ cert_fileZkey_fileZcert)rrr')r ssl_detailsZssl_argsrr!r!r" _get_ssl_args[s   rrTFr)streamr.c st|}||d}|t||| |d<|s:|r6dnd}||d<|durpt|tr^||d<ntt|d|d<|dur|g}d }|rtt|d d }d t }|dur| }ni}|r||d <|durd }| durt } t D] }|r||}d |vr ||d <||d<i}|D]p\}|d kr4q|dkr|rfdd|D}|rt ||<|D]}t|||<qnn||<qzl|rtd|| rdn|||| jfi|}| r|td||jt|j|d t|WStjy2}zt||d|WYd}~qd}~0tjy}zt|tjrt|drt|jdrt||jj|jj |d}n t||d}| r| | |s||| p|d |k}|s|||dkr|rtd|t!"|WYd}~qd}~00qt#ddS)aWrapper around requests.Session to read the url and retry if necessary :param url: Mandatory url to request. :param data: Optional form data to post the URL. Will set request_method to 'POST' if present. :param timeout: Timeout in seconds to wait for a response. May be a tuple if specifying (connection timeout, read timeout). :param retries: Number of times to retry on exception if exception_cb is None or exception_cb returns True for the exception caught. Default is to fail with 0 retries on exception. :param sec_between: Default 1: amount of seconds passed to time.sleep between retries. None or -1 means don't sleep. :param headers: Optional dict of headers to send during request :param headers_cb: Optional callable returning a dict of values to send as headers during request :param headers_redact: Optional list of header names to redact from the log :param ssl_details: Optional dict providing key_file, ca_certs, and cert_file keys for use on in ssl connections. :param check_status: Optional boolean set True to raise when HTTPError occurs. Default: True. :param allow_redirects: Optional boolean passed straight to Session.request as 'allow_redirects'. Default: True. :param exception_cb: Optional callable which accepts the params msg and exception and returns a boolean True if retries are permitted. :param session: Optional exiting requests.Session instance to reuse. :param infinite: Bool, set True to retry indefinitely. Default: False. :param log_req_resp: Set False to turn off verbose debug messages. :param request_method: String passed as 'method' to Session.request. Typically GET, or POST. Default: POST if data is provided, GET otherwise. :param stream: if False, the response content will be immediately downloaded. )r rallow_redirectsZPOSTZGETmethodNrArrz Cloud-Init/%sr^z User-AgentrEcsg|]}|r|qSr!)r5).0kvr!r" rzreadurl..z'[%s/%s] open '%s' with %s configurationinfinitez(Read from %s (%s, %sb) after %s attemptsr rrr=rEr 1Please wait %s seconds while we wait to try againz"This path should be unreachable...)$r#updaterr7tuplemaxfloatrrZversion_stringcopyrZSessionritemsdeepcopyrr:rTZrequestZraise_for_statusrlenrrcrZSSLErrorrPZRequestExceptionZ HTTPErrorhasattrrrEtimesleep RuntimeError)r r^rAZretriesZ sec_betweenrE headers_cbheaders_redactr check_statusr exception_cbZsessionrZ log_req_resprequest_methodrZreq_argsZ manual_triesZ user_agentiZfiltered_req_argsrZmatched_headerskeyrrZZ url_errorZ will_retryr!rr"rhms4           "   "rh.)funcaddrrAeventdelayr.cCs|r|j|drdS|||S)z Execute func with optional delayrANwaitrrrArrr!r!r"_run_func_with_delays r333333? )r addresses stagger_delayrAr.c s8d}d}d}g}ttt|dzzfddt|D}t|dD]X} || }| } | r|| }||qT| }|rT ||fWWj ddSqT|rt d|||nt d |td Wn4tyt d d |d tt|Yn0Wj ddnj dd0||fS) aexecute multiple callbacks in parallel Run blocking func against two different addresses staggered with a delay. The first call to return successfully is returned from this function and remaining unfinished calls are cancelled if they have not yet started N) max_workersc s,i|]$\}}jt||d|qS)r)Zsubmitr)rrrexecutorrZis_donerrAr!r" Is zdual_stack..rFrz.default_sleep_timerr start_timercSs0|tddfvrdS|dkp.t|||kS)z4Check if time is up based on start time and max waitrNFr)rr monotonicrr!r!r"timeups zwait_for_url..timeupcSsf|js*d|j}tt||j|j|d}n4|sVd|j}tt||j|j|d}nd}d}||fS)z?Map requests response code/contents to internal "UrlError" typezempty response [%s]rzbad status code [%s]rN)rar=rPrgrErt)rr reasonurl_excr!r!r"handle_url_responses&  z)wait_for_url..handle_url_responsec sd}d}z,||\}}||\}}|s2||fWSWn^tyd} zd| }| }WYd} ~ n8d} ~ 0ty} zd| }| }WYd} ~ n d} ~ 00tt|} rdnd} d|pt|dd| | |f} || |r|| |d dS) z:Execute request, handle response, optionally log exceptionrNzrequest error [%s]zunexpected error [%s]z%ssZ unlimitedzCalling '%s' failed [%s/%s]: %sr )msgr)rP Exceptionrrrgetattr) Z url_reader_cburlsrexc_cblog_cbrr rrrZZ time_takenZ max_wait_strZ status_msg)rrr!r"read_url_handle_exceptionss0 z0wait_for_url..read_url_handle_exceptionscs$t|durin||ddS)NF)rErrArr)rh)r rA)rrrr!r" read_url_cbsz!wait_for_url..read_url_cbcsfdd}D]l}t}dkrb|r6dSdurbrb||krbt|||||||}|r|SqdS)z|iterate over list of urls, request each one and handle responses and thrown exceptions individually per url cs||fSrmr!r)rrAr!r"url_reader_serial sz@wait_for_url..read_url_serial..url_reader_serialrN)rrr)rrArrrr Znowout)loop_nrrrrrrr"read_url_serials$  z%wait_for_url..read_url_serialcs,tt|d}||||}|r(|SdS)zpass list of urls to dual_stack which sends requests in parallel handle response and exceptions of the first endpoint to respond )rrAN)rr)rrArrZurl_reader_parallelr)rrrrr!r"read_url_parallel"s z'wait_for_url..read_url_parallelz3sleep_time and sleep_time_cb are mutually exclusiveNrrz$Timed out, no response from urls: %s)FN)r) rrrrrgrar:rTrr)rrrArrrrrrrrrrrrrZ do_read_urlZcalculate_sleep_timerZcurrent_sleep_timer addressZ current_timer!) rrrrrrrrrrrrr" wait_for_urlsF9         rc@s^eZdZdddZddZddZd d Zd d Zd dZddZ ddZ ddZ ddZ dS)OauthUrlHelperN/run/oauth_skew.jsoncCst||_|p d|_||_||_||_d|_d|_|j|j|jf}t|sNd|_nt|s^t d| }|pli|_ dS)NrTrFzBall or none of token_key, token_secret, or consumer_key can be set) consumer_keyconsumer_secret token_key token_secretskew_data_file _do_oauthskew_change_limitanyallrgread_skew_file skew_data)rorrrrrrequiredoldr!r!r"rpbs  zOauthUrlHelper.__init__c Cs|jrtj|jrtd|jVt|jdd(}t|WdWdS1sd0YWdn1s0YdS)NzReading rmode) rosr)isfilerTimedopenjsonload)rofpr!r!r"r}sTzOauthUrlHelper.read_skew_filec Cs|js dS|}|duri}|||<td|jNt|jdd }|t|Wdn1sn0YWdn1s0YdS)NzWriting wr)rrrrrrVrdumps)rorIvalueZcurr r!r!r"update_skew_fileszOauthUrlHelper.update_skew_filec Cst|tr|jdks"|jdks"dSd|jvr>td|jdS|jd}ztt|}Wn4t y}ztd||WYd}~dSd}~00t |t}t |j j }|j|d}t|||jkr|||td||||j|<dS)Niidatez$Missing header 'date' in %s responsez#Failed to convert datetime '%s': %srz$Setting oauth clockskew for %s to %d)r7rPr=rEr:r;rmktimerrrrr netlocrr5absrr ) rorrrZ remote_timerZZskewrIZold_skewr!r!r"rs0     zOauthUrlHelper.exception_cbcCsZ|js iSd}t|j}|jr>||jvr>tt|j|}t||j|j|j |j |dS)N)r rrrr timestamp) rrrrrr oauth_headersrrrr)ror rrIr!r!r"rs zOauthUrlHelper.headers_cbcCs:t|j|d|d<t|j|d|d<||i|S)Nrr)r _headers_cbr5 _exception_cb)roZ wrapped_funcrsrBr!r!r"_wrappeds  zOauthUrlHelper._wrappedcOs|t||Srm)rrrrr!r!r"rszOauthUrlHelper.wait_for_urlcOs|t||Srm)rrhrrr!r!r"rhszOauthUrlHelper.readurlc Cs6d}z|r|||}W|||n|||0|Srm)r)roZextra_exception_cbrrretr!r!r"rs  zOauthUrlHelper._exception_cbcCs$i}|r||}||||Srm)rr)roZextra_headers_cbr rEr!r!r"rs zOauthUrlHelper._headers_cb)NNNNr) rxryrzrprr rrrrrhrrr!r!r!r"ras    rc Cs~zddlm}Wn.ty>}ztd|WYd}~n d}~00|rNt|}nd}|j|||||j|d}||\} } } | S)Nrzoauth support is not available)Z client_secretZresource_owner_keyZresource_owner_secretZsignature_methodr)Zoauthlib.oauth1oauth1 ImportErrorNotImplementedErrorr'ZClientZSIGNATURE_PLAINTEXTsign) r rrrrrrrZZclientZ_uriZsigned_headersZ_bodyr!r!r"rs"  r)r?)NNrrNNNNTTNNFTrF)N)rr)N)Grr2rQrZloggingrrrconcurrent.futuresrrrZ email.utilsr functoolsrZ http.clientr itertoolsrZsslr typingr r r r rrr urllib.parserrrrrrZ cloudinitrrrZ getLoggerrxr:rr#r,rr>r'rdictr\rbrjrkr]r@rcr`rPrrrhrrrrTrrrr!r!r!r" s     $    1  6    U `