a 'DgQ@sUdZddlZddlZddlZddlZddlZddlZddlmZddl m Z m Z m Z m Z ddlmZmZmZmZmZddlmZddlmZddlmZdd lmZdd lmZeeZ d Z!d Z"d Z#dZ$dZ%dddZ&e e'e'fe(d<dddgegdZ)ee(d<dZ*dZ+dZ,dZ-dddZ.d d dZ/d!d"gZ0gd#Z1d$Z2de2iZ3dud%d&Z4e'eee5dd'd(d)Z6d*d+Z7d,d-Z8d.d/Z9d0d1Z:d2d3Z;d4d5Zd:d;Z?e'e'd<d=d>Z@e'd?d@dAZAe'd?dBdCZBdDdEZCe'eDdFdGdHZEdIdJdKdLZFdMZGe e'e'fd?dNdOZHdPdQZIdvdSdTZJdUdVZKdwdWdXZLdxdYdZZMd[d\ZNd]d^ZOd_d`ZPdadbZQdcddZRdedfZSdgdhZTdidjZUdkdlZVdydmdnZWdodpZXdzdrdsZYdtez,apply_debconf_selections..#z[:\s].*z pkgs_cfgd: %srzno need for reconfig)r,r@rAjoinsortedkeysr\encodesetitems splitlines startswithrDsubaddr Zget_installed_packages intersectionr_rc) r&r[Z pkgs_cfgdZ_keyZcontentlinerbZpkgs_installedZ need_reconfigr"rgr#r1s(          r1cCs6ttjdd}td||D]}t|q"dS)z%clean out any local cloud-init configz/etc/cloud/cloud.cfg.d/*dpkg*pathz#cleaning cloud-init config from: %sN)globr target_pathr@rAosunlink)flistZdpkg_cfgr"r"r#clean_cloud_inits rcCsL|}|dr|dd}|d}|dkr<||dd}|dd}|S)zmirrorurl_to_apt_fileprefix Convert a mirror url to the file prefix used by apt on disk to store cache information for that mirror. To do so do: - take off ???:// - drop tailing / - convert in string / to _/rz://N_)rZfindreplace)mirrorstringposr"r"r#mirrorurl_to_apt_fileprefixs    rc Cst|}tt}|D]\}}||}|s2q|tjjt |}|tjjt |}||krdqt |} t d|D]X} d|| | df} t d| | zt| | Wqztyt jdddYqz0qzqdS)z>rename_apt_lists - rename apt lists to preserve old cache dataz%s_*z%s%sNzRenaming apt list %s to %szFailed to rename apt list:T)exc_info)r$r r{ APT_LISTSrqr,r|ryseprr_rzr@rArenamerPr`) Z new_mirrorsr!Zdefault_mirrorsZprer%ZomirrorZnmirrorZoprefixZnprefixZolenfilenameZnewnamer"r"r#rKs$   rKcCs8dddddd}z ||}Wnty2|}Yn0|S)zthere are a few default names which will be auto-extended. This comes at the inability to use those names literally as suites, but on the other hand increases readability of the cfg quite a lotz$RELEASE-updatesz$RELEASE-backportsz$RELEASE-securityz$RELEASE-proposedz$RELEASE)ZupdatesZ backportssecurityZproposedrT)KeyError)suitemappingZretsuiter"r"r#map_known_suites/s   r) deb822_entryr)cCs<td|s8tdd|}ttd|}dt|dS|S)z0If no active Suites, disable this deb822 source.z\nSuites:[ \t]+([\w-]+)z \nSuites:.*rkz7## Entry disabled by cloud-init, due to disable_suites z# disabled by cloud-init: )rDfindallrtDISABLE_SUITES_REDACT_PREFIXr)rr"r"r#%disable_deb822_section_without_suitesAs r)r)c s$g}fdd|Dtd|d}|D]}|dr^|rR||d7}q0||q0|rj|r|r|t|d}||q0|}|ds||d7}q0r|dd }fd d|D}||kr|t|d7}d d |}||d7}q0|r|t|d |S) z:reads the deb822 format config and comment disabled suitescs g|]}tt|diqS)r<)r render_stringrrer)rTr"r#rhTsz)disable_suites_deb822..zDisabling suites %s as %srkrjrdzSuites:Ncsg|]}|vr|qSr"r"r)disabled_suite_namesr"r#rhrszSuites:  ) r@rArrrsr^isspacersplitrrl) disabledsrcrTZnew_srcZnew_deb822_entryrwZnew_lineZ orig_suitesZ new_suitesr")rrTr#disable_suites_deb822QsH         rc Cs|s|S|}t|r t|||S|D]}t|}t|d|i}td||d}|dD]}|drt||7}q\| }t |dkrd} |ddr|dd D]} | d7} | d rqq|| |krd |}||7}q\|}q$|S) zRreads the config for suites to be disabled and removes those from the templater<zDisabling suite %s as %srkTrjr[N]z"# suite disabled by cloud-init: %s) is_deb822_sources_formatrrr rr@rArrrsrr_rZ) rrrTZretsrcrZ releasesuiteZnewsrcrwZcolsZpcolcolr"r"r#disable_suitess6      rcCs0dD]&}||gD]}t||||dqqdS)z=Adds any keys included in the primary/security mirror clausesprimaryr) file_nameN)r, add_apt_key)r&r'rrfrr"r"r#rIsrI)apt_src_contentr)cCs6td|tjrdStd|tjr(dStddS)a?Simple check for deb822 format for apt source content Only validates that minimal required keys are present in the file, which indicates we are likely deb822 format. Doesn't handle if multiple sections all contain deb822 keys. Return True if content looks like it is deb822 formatted APT source. z^(deb |deb-src )Fz'^(Types: |Suites: |Components: |URIs: )Tzapt.sources_list value does not match either deb822 source keys or deb/deb-src list keys. Assuming APT deb/deb-src list format.)rDrMr@r`)rr"r"r#rs rzetc/apt sources.listzsources.list.d)Dir::EtcDir::Etc::sourcelistDir::Etc::sourcepartsz@(Dir::Etc|Dir::Etc::sourceparts|Dir::Etc::sourcelist) \"([^\"]+)c Cs4zJddl}||jdtd}|jdtd}|jdtd}Wntyztddg\}}WnTtjytd}td}td}d|d|d|d|dd YYS0t t |}t |}|dtd}|dtd}|dtd}Yn0d|d|d|d|dd S) aReturn a dict of applicable apt configuration or defaults. Prefer python apt_pkg if present. Fallback to apt-config dump command if present out output parsed Fallback to DEFAULT_APT_CFG if apt-config command absent or output unparsable. rNrrrz apt-configdumpr) sourcelist sourceparts) apt_pkgZ init_configZconfigr,DEFAULT_APT_CFG ImportErrorr ProcessExecutionErrorrDr APT_CFG_REr.)retcrrZapt_dumprZ matched_cfgZapt_cmd_configr"r"r# get_apt_cfgs@  rcCst}|d}|d|jjd}tjr0|}n|}||d}|D] } || || <|| || <qB|dd} | stdtjrdnd } | d |jj| } | s| d } | st d |dSt | } t | |} | rt| r||krtd ||}ntd||}t|d| |}t j||dd||krtj|rt|jj}|r|t |krtd|t |tntd|t |dS)zgenerate_sources_list create a source.list file based on a custom or default template by replacing mirrors and release in the templaterrz.sources)r<r8 sources_listNz1No custom template provided, fall back to builtinz.deb822rkz sources.list.rz#No template found, not rendering %szAProvided 'sources_list' user-data is deb822 format, writing to %szFProvided 'sources_list' user-data is not deb822 format, fallback to %sri)modez*Replacing %s to favor deb822 source formatz)Removing %s to favor deb822 source format)rdistror%rZAPT_DEB822_SOURCE_LIST_FILElowerr,r@infoZget_template_filenamer`r Zload_text_filer rrrAr write_filer|ryexistsDEB822_ALLOWED_APT_SOURCES_LISTUBUNTU_DEFAULT_APT_SOURCES_LISTdel_file)r&rTrUr'r3Zapt_sources_listZapt_sources_deb822Z aptsrc_filerVkZtmplZtmpl_fmtZ template_fnZrenderedrZexpected_contentr"r"r#rJsp          rJFcCsRtd|z t|j}td||||dWStjyLtdYn0dS)zM actual adding of a key as defined in key argument to the system zAdding key: '%s'ru) output_filerYhardenedz(failed to add apt GPG Key to apt keyringN) r@rApathlibPathstemapt_keyr rrQ)rfrrrr%r"r"r#add_apt_key_rawLs    rc Csg}t}t|ddrVdD]4}||r ||D]}ddh|r6|dq6q |di}|D]6}ddh|r|d||dd rj|d qj|D]} t| s| t | q|r|j t |d S) aInstall missing package dependencies based on apt_sources config. Inspect the cloud config user-data provided. When user-data indicates conditions where add_apt_key or add-apt-repository will be called, ensure the required command dependencies are present installed. Perform this inspection upfront because it is very expensive to call distro.install_packages due to a preliminary 'apt update' called before package installation. r:Frrfkeyidrr;sourcerkrN)rpr rHr,rvruvaluesshutilr6r^rrZinstall_packagesrm) r&r?r'Zmissing_packagesZ required_cmdsZ mirror_keyZ mirror_itemZapt_sources_dictentcommandr"r"r#rG\s&        rGcCs`d|vr8d|vr8t}d|vr$|d}||d||d<d|vr\t|d|pR|d||dSdS)z Add key to the system as defined in ent (if any). Supports raw keys or keyid's The latter will as a first step fetched to get the raw key rrf keyserverrrN)DEFAULT_KEYSERVERZ getkeybyidr)rr'rrrrr"r"r#r~src Cs|dur i}|durtdt|ts2td||D]}||}td|d|vr`||d<d|vrd|dvrt|||dd }||d <n t|||d|vrq6|d}t||}|d d st j d |d|d<|d d s|dd 7<||r>ztdd|gWq6tjy:tdYq60q6tj|dd} z8d|} d} d|vrt|dstd} tj| | | dWq6ty} ztd| | WYd} ~ q6d} ~ 00q6|jjdddS)a install keys and repo source .list files defined in 'sources' for each 'source' entry in the config: 1. expand template variables and write source .list file in /etc/apt/sources.list.d/ 2. install defined keys 3. update packages via distro-specific method (i.e. apt-key update) @param srcdict: a dict containing elements required @param cloud: cloud instance object Example srcdict value: { 'rio-grande-repo': { 'source': 'deb [signed-by=$KEY_FILE] $MIRROR $RELEASE main', 'keyid': 'B59D 5F15 97A5 04B7 E230 6DCA 0620 BBCF 0368 3F77', 'keyserver': 'pgp.mit.edu' } } Note: Deb822 format is not supported Nz did not get a valid repo matcherzunknown apt format: %szadding source/key '%s'rrz $KEY_FILETrZKEY_FILErz/etc/apt/sources.list.d/z.listrz --no-updatezadd-apt-repository failed.rxz%s ar^w)omodezfailed write to file %s: %s)force)rr-r. TypeErrorr@rArr rrsr|ryrlrZr rrQr{r rrOrZupdate_package_sources) srcdictr'rr>r?rrkey_filerZsourcefncontentsrZdetailr"r"r#rRs\         rRcCs~i}tjddddt|trbtd|D]2}d|vrNd|d<t|d}n|d}|||<q,nt|trr|}nt d|S) z1convert v1 apt format to v2 (dict in apt_sources)zConfig key 'apt_sources'22.1zUse 'apt' instead) deprecateddeprecated_versionZ extra_messagez9apt config: convert V1 to V2 format (source list to dict)rzcloud_config_sources.listzunknown apt_sources format) r deprecater-rar@rAr Z rand_dict_keyr.r)ZsrclistrZsrcentrfr"r"r#convert_v1_to_v2_apt_formats$    rcCs,||ddur(||||<||=dSdS)ziconvert an old key to the new one if the old one exists returns true if a key was found and convertedNTFr,)oldcfgaptcfgoldkeynewkeyr"r"r# convert_keys rcCsHgd}d}ddgi}|D]\}}t||||rd}q|rD|g|d<dS)zBconvert old apt_mirror keys into the new more advanced mirror spec)) apt_mirroruri)apt_mirror_searchrF)apt_mirror_search_dns search_dnsFarchesdefaultTrN)r)rrZkeymapZ convertedZnewmcfgrrr"r"r#convert_mirrors  rc CsZddddddddddd d }g}|D]*}||vr$||d vrD||=q$||q$|sX|Stjd |d d|dd}|durtjdd d|D]V}||}||}||=|dus||ddurq|||krtd||||fq|Si}|D]"}||durt|||||qt|||D]$}||ddur(td|q(||d<|S)z:convert old to new keys and adapt restructured mirror specr;Nproxy http_proxy https_proxy ftp_proxyr:rr9) apt_sourcesrrrZ apt_proxyZapt_http_proxyZ apt_ftp_proxyZapt_https_proxyZapt_preserve_sources_listZapt_custom_sources_listr9)NrkzThe following config key(s): r)rrr*z0Support for combined old and new apt module keysz@Old and New apt format defined with unequal values %s vs %s @ %sz&old apt key '%s' left after conversion)r^rrr,rrr)rZ mapoldkeysZ needtoconvertrZ newaptcfgrZverifyrr"r"r#convert_v2_to_v3_apt_format!sj        rcCs,|dd}|dur t||d<t|}|S)zconvert the old list based format to the new dict based one. After that convert the old dict keys/format to v3 a.k.a 'new apt config'rN)r,rr)r&rr"r"r#r+ks   r+c Csd}|rd}g}|dkrd}n|dkr,d}ntdt||j}d|dd d}|rl|d ||d g} |jj } d | |d | f} |D]} | | | qt | }|S)zG Try to resolve a list of predefines DNS names to pick mirrors Nrkrrrzsecurity-mirrorzunknown mirror type.rz.%s)z .localdomainrkzhttp://%s-%s%s/%sz%s) rr Zget_hostname_fqdnfqdnrlrr^extendrr%search_for_mirror) Z configured mirrortyper&r'rZmydomZdomsZ mirrordnsrZ mirror_listrZ mirrorfmtZpostr"r"r#search_for_mirror_dnszs. rcCsX|dur|dur|}||dS|j}|rP|}|d|d<|d|d<|St|S)z^sets security mirror to primary if not defined. returns defaults if no mirrors are definedNrrrrr)Z datasourceZget_package_mirror_inforr$)pmirrorsmirrorr!r' mirror_infomr"r"r#update_mirror_infos    rcCsT||d}|durdSd}|D].}|dp0g}||vrB|Sd|vr |}q |S)zuout of a list of potential mirror configurations select and return the one matching the architecture (or default)Nrrr)r&rr!Zmirror_cfg_listrZmirror_cfg_elemrr"r"r#get_arch_mirrorconfigs rcCs`t|||}|durdS|dd}|dur>t|dd}|dur\t|dd|||}|S)zpass the three potential stages of mirror specification returns None is neither of them found anything otherwise the first hit is returnedNrrFr)rr,r rr)r&rr!r'Zmcfgrr"r"r# get_mirrors  rcCsn|durt}td|t|d||}td|t|d||}td|t||||}|d|d<|S) afind_apt_mirror_info find an apt_mirror given the cfg provided. It can check for separate config of primary and security mirrors If only primary is given security is assumed to be equal to primary If the generic apt_mirror is given that is defining for both Nz!got arch for mirror selection: %srzgot primary mirror: %srzgot security mirror: %srr=)r rr@rArr)r&r'r!rrrr"r"r#rBs    rBcsd}fdd|D}t|rBtd|t|d|dn"tj|rdt |td| ddrtd |t| dn"tj|rt |td |dS) zHapply_apt_config Applies any apt*proxy config from if specified ))rAcquire::http::Proxy "%s";)rr)rzAcquire::ftp::Proxy "%s";)rzAcquire::https::Proxy "%s";cs(g|] \}}|r||qSr"r)rer%fmtr&r"r#rh riz$apply_apt_config..zwrite apt proxy info to %srdz#no apt proxy configured, removed %sZconfNzwrite apt config info to %sz$no apt config configured, removed %s) r_r@rAr rrlr|ryisfilerr,)r&Z proxy_fnameZ config_fnameZcfgsZproxiesr"rr#rLs        rLTcsZddfdd}fdd}|dkr6||S|dksF|d krN||Std d S) adapt-key replacement commands implemented: 'add', 'list', 'finger' @param output_file: name of output gpg file (without .gpg or .asc) @param data: key contents @param human_output: list keys formatted for human parsing @param hardened: write keys to to /etc/apt/cloud-init.gpg.d/ (referred to with [signed-by] in sources file) cSsJtjtrtgng}ttD]}|dr |t|q |rF|SdS)zreturn all apt keys /etc/apt/trusted.gpg (if it exists) and all keyfiles (and symlinks to keyfiles) in /etc/apt/trusted.gpg.d/ are returned based on apt-key implementation )z.gpgz.ascrk)r|ryrAPT_LOCAL_KEYSlistdirAPT_TRUSTED_GPG_DIRrZr^)Z key_filesfiler"r"r#_get_key_files/s  zapt_key.._get_key_filesc sd}sttdn|z2r&tnt}|}d|}t||WnHtj ytttdYn$t yttdYn0|S)ziapt-key add returns filepath to new keyring, or '/dev/null' when an error occurs z /dev/nullz)Unknown filename, failed to add key: "{}"z{}{}.gpgz Gpg error, failed to add key: {}z#Decode error, failed to add key: {}) r Zlogexcr@r/CLOUD_INIT_GPG_DIRr Zdearmorrr rUnicodeDecodeError)r4rZkey_dirstdout)rYrrr"r# apt_key_add>s(       zapt_key..apt_key_addc shg}D]R}z||j|dWq tjyZ}ztd||WYd}~q d}~00q d|S)zapt-key list returns string of all trusted keys (in /etc/apt/trusted.gpg and /etc/apt/trusted.gpg.d/) ) human_outputzFailed to list key "%s": %sNrd)r^Z list_keysr rr@r`rl)r4Zkey_listrerror)r rr"r# apt_key_listZs  &zapt_key..apt_key_listruZfingerraz@apt_key() commands add, list, and finger are currently supportedN)r)rrrrYrrrrr")r rYrrrr#rsrz cloud-init)N)F)FN)NN)N)NNFT)[__doc__rzZloggingr|rrDrtextwraprtypingrrrrZ cloudinitrrr r r Zcloudinit.cloudr Zcloudinit.configr Zcloudinit.config.schemarZ cloudinit.gpgrZcloudinit.settingsrZ getLogger__name__r@rCr r rrrstr__annotations__rrrNrMrrrrrrrr$rar5r7r2r\rcr1rrrKrrrrrIboolrrrrrJrrGrrRrrrrr+rrrrrBrLrr]r"r"r"r# s           .  # /(3F "  X J)  Y