a Á]›iëéã@s"ddlmZddlTddlZddlZddlTddlZz ddlZWne yRYn0ddl Z ddl m Z ddl Z ddlmZddlZddlZddlZddlZz ejWneyÀddlZYn0ej ¡ZGdd„deƒZGdd „d eƒZGd d „d eƒZGd d „d eƒZGdd„dejƒZ dS)é)Úbase)Ú*éN)Úcommandsc@s0eZdZddd„Zedd„ƒZejdd„ƒZdS)ÚSchedulerParamsNcCs(||_||_||_||_||_||_dS©N)Ú_cmdÚcmdlineÚ schedulerÚpriorityÚaffinityÚcgroup)ÚselfÚcmdr r r r r ©rúB/usr/lib/python3.9/site-packages/tuned/plugins/plugin_scheduler.pyÚ__init__"s zSchedulerParams.__init__cCs |jdurdS|j |j¡SdSr)Ú _affinityrZbitmask2cpulist©rrrrr +s zSchedulerParams.affinitycCs"|durd|_n|j |¡|_dSr)rrZcpulist2bitmask©rÚvaluerrrr 2s)NNNNN)Ú__name__Ú __module__Ú __qualname__rÚpropertyr Úsetterrrrrr!sÿ  rc@seZdZdd„ZdS)Ú IRQAffinitiescCsi|_d|_g|_dSr)ÚirqsÚdefaultÚ unchangeablerrrrr:szIRQAffinities.__init__N)rrrrrrrrr9src@speZdZdZddddddœZdd „Zd d „Zd d „Zdd„Zdd„Z dd„Z dd„Z dd„Z dd„Z dd„ZdS)ÚSchedulerUtilsz= Class encapsulating scheduler implementation in os module Ú SCHED_FIFOÚ SCHED_BATCHÚSCHED_RRÚ SCHED_OTHERÚ SCHED_IDLE)ÚfÚbÚrÚoÚicCs8tdd„|j ¡Dƒƒ|_tdd„|j ¡Dƒƒ|_dS)Ncss |]\}}|tt|ƒfVqdSr©ÚgetattrÚos©Ú.0ÚkÚnamerrrÚ Oóz*SchedulerUtils.__init__..css|]}tt|ƒ|fVqdSrr+©r/r1rrrr2Qr3©ÚdictÚ_dict_schedcfg2schedconstÚitemsÚ_dict_schedcfg2numÚvaluesÚ_dict_num2schedconstrrrrrMszSchedulerUtils.__init__cCs |j |¡Sr)r9Úget)rÚ str_schedulerrrrÚsched_cfg_to_numSszSchedulerUtils.sched_cfg_to_numcCs |j |¡Sr)r;r<)rr rrrÚsched_num_to_constWsz!SchedulerUtils.sched_num_to_constcCs t |¡Sr)r-Úsched_getscheduler©rÚpidrrrÚ get_schedulerZszSchedulerUtils.get_schedulercCst ||t |¡¡dSr)r-Úsched_setschedulerÚ sched_param©rrBÚschedÚpriorrrÚ set_scheduler]szSchedulerUtils.set_schedulercCs t |¡Sr)r-Úsched_getaffinityrArrrÚ get_affinity`szSchedulerUtils.get_affinitycCst ||¡dSr)r-Úsched_setaffinity©rrBr rrrÚ set_affinitycszSchedulerUtils.set_affinitycCs t |¡jSr)r-Úsched_getparamÚsched_priorityrArrrÚ get_priorityfszSchedulerUtils.get_prioritycCs t |¡Sr)r-Úsched_get_priority_min©rrGrrrÚget_priority_miniszSchedulerUtils.get_priority_mincCs t |¡Sr)r-Úsched_get_priority_maxrSrrrÚget_priority_maxlszSchedulerUtils.get_priority_maxN)rrrÚ__doc__r7rr>r?rCrIrKrNrQrTrVrrrrr @s"ûr c@sPeZdZdZdd„Zdd„Zdd„Zdd „Zd d „Zd d „Z dd„Z dd„Z dS)ÚSchedulerUtilsSchedutilszE Class encapsulating scheduler implementation in schedutils module cCs8tdd„|j ¡Dƒƒ|_tdd„|j ¡Dƒƒ|_dS)Ncss |]\}}|tt|ƒfVqdSr©r,Ú schedutilsr.rrrr2ur3z4SchedulerUtilsSchedutils.__init__..css|]}tt|ƒ|fVqdSrrYr4rrrr2wr3r5rrrrrssz!SchedulerUtilsSchedutils.__init__cCs t |¡Sr)rZrCrArrrrCysz&SchedulerUtilsSchedutils.get_schedulercCst |||¡dSr)rZrIrFrrrrI|sz&SchedulerUtilsSchedutils.set_schedulercCs t |¡Sr)rZrKrArrrrKsz%SchedulerUtilsSchedutils.get_affinitycCst ||¡dSr)rZrNrMrrrrN‚sz%SchedulerUtilsSchedutils.set_affinitycCs t |¡Sr)rZrQrArrrrQ…sz%SchedulerUtilsSchedutils.get_prioritycCs t |¡Sr)rZrTrSrrrrTˆsz)SchedulerUtilsSchedutils.get_priority_mincCs t |¡Sr)rZrVrSrrrrV‹sz)SchedulerUtilsSchedutils.get_priority_maxN) rrrrWrrCrIrKrNrQrTrVrrrrrXosrXcsFeZdZdZddddœZdd„Z‡fdd„Zd d „Zd d „Zd d„Z e dd„ƒZ dd„Z dd„Z dd„Zdd„Zdd„Zdd„Zdd„Zdd „Zd!d"„Zd#d$„Zd%d&„ZdØd(d)„Zd*d+„Zd,d-„Zd.d/„ZdÙd0d1„Zd2d3„Zd4d5„Zd6d7„Zd8d9„Zd:d;„Z dd?„Z"d@dA„Z#dBdC„Z$dDdE„Z%dÚdFdG„Z&dHdI„Z'dJdK„Z(‡fdLdM„Z)dNdO„Z*dPdQ„Z+dRdS„Z,e-j.f‡fdTdU„ Z/dVdW„Z0dXdY„Z1‡fdZd[„Z2d\d]„Z3d^d_„Z4d`da„Z5e6dbd'dc�ddde„ƒZ7e6dfd'dc�dgdh„ƒZ8e6did'dc�djdk„ƒZ9e6dld'dc�dmdn„ƒZ:e6dod'dc�dpdq„ƒZ;e6drd'dc�dsdt„ƒZdzd{„Z?d|d}„Z@dÛd~d„ZAd€d�„ZBd‚dƒ„ZCd„d…„ZDd†d‡„ZEdˆd‰„ZFdŠd‹„ZGdŒd�„ZHdŽd�„ZId�d‘„ZJe6d’d'd“d”�d•d–„ƒZKd—d˜„ZLd™dš„ZMd›dœ„ZNd�dž„ZOdÜdŸd „ZPeQd¡ƒd¢d£„ƒZReSd¡ƒd¤d¥„ƒZTeQd¦ƒd§d¨„ƒZUeSd¦ƒd©dª„ƒZVeQd«ƒd¬d­„ƒZWeSd«ƒd®d¯„ƒZXeQd°ƒd±d²„ƒZYeSd°ƒd³d´„ƒZZeQdµƒd¶d·„ƒZ[eSdµƒd¸d¹„ƒZ\eQdºƒd»d¼„ƒZ]eSdºƒd½d¾„ƒZ^eQd¿ƒdÀdÁ„ƒZ_eSd¿ƒdÂdăZ`eQdădÅdÆ„ƒZaeSdădÇdÈ„ƒZbeQdɃdÊdË„ƒZceSdɃdÌdÍ„ƒZdeQd΃dÏdЄƒZeeSd΃dÑdÒ„ƒZfeQdÓƒdÔdÕ„ƒZgeSdÓƒdÖdׄƒZh‡ZiS)ÝÚSchedulerPluginaX. Allows tuning of scheduling priorities, process/thread/IRQ affinities, and CPU isolation. To prevent processes/threads/IRQs from using certain CPUs, use the [option]`isolated_cores` option. It changes process/thread affinities, IRQs affinities and it sets `default_smp_affinity` for IRQs. The CPU affinity mask is adjusted for all processes and threads matching [option]`ps_whitelist` option subject to success of the `sched_setaffinity()` system call. The default setting of the [option]`ps_whitelist` regular expression is `.*` to match all processes and thread names. To exclude certain processes and threads use [option]`ps_blacklist` option. The value of this option is also interpreted as a regular expression and process/thread names (`ps -eo cmd`) are matched against that expression. Profile rollback allows all matching processes and threads to run on all CPUs and restores the IRQ settings prior to the profile application. Multiple regular expressions for [option]`ps_whitelist` and [option]`ps_blacklist` options are allowed and separated by `;`. Quoted semicolon `\;` is taken literally. .Isolate CPUs 2-4 ==== ---- [scheduler] isolated_cores=2-4 ps_blacklist=.*pmd.*;.*PMD.*;^DPDK;.*qemu-kvm.* ---- Isolate CPUs 2-4 while ignoring processes and threads matching `ps_blacklist` regular expressions. ==== The [option]`irq_process` option controls whether the scheduler plugin applies the `isolated_cores` parameter to IRQ affinities. The default value is `true`, which means that the scheduler plugin will move all possible IRQs away from the isolated cores. When `irq_process` is set to `false`, the plugin will not change any IRQ affinities. The [option]`default_irq_smp_affinity` option controls the values *TuneD* writes to `/proc/irq/default_smp_affinity`. The file specifies default affinity mask that applies to all non-active IRQs. Once an IRQ is allocated/activated its affinity bitmask will be set to the default mask. The following values are supported: * `calc` + The content of `/proc/irq/default_smp_affinity` will be calculated from the `isolated_cores` parameter. Non-isolated cores are calculated as an inversion of the `isolated_cores`. Then the intersection of the non-isolated cores and the previous content of `/proc/irq/default_smp_affinity` is written to `/proc/irq/default_smp_affinity`. If the intersection is an empty set, then just the non-isolated cores are written to `/proc/irq/default_smp_affinity`. This behavior is the default if the parameter `default_irq_smp_affinity` is omitted. * `ignore` + *TuneD* will not touch `/proc/irq/default_smp_affinity`. * an explicit cpulist + The cpulist (such as `1,3-4`) is unpacked and written directly to `/proc/irq/default_smp_affinity`. .An explicit CPU list to set the default IRQ smp affinity to CPUs 0 and 2 ==== ---- [scheduler] isolated_cores=1,3 default_irq_smp_affinity=0,2 ---- ==== To adjust scheduling policy, priority and affinity for a group of processes/threads, use the following syntax. [subs="quotes"] ---- group.__groupname__=__rule_prio__:__sched__:__prio__:__affinity__:__regex__ ---- Here, `__rule_prio__` defines internal *TuneD* priority of the rule. Rules are sorted based on priority. This is needed for inheritence to be able to reorder previously defined rules. Equal `__rule_prio__` rules should be processed in the order they were defined. However, this is Python interpreter dependant. To disable an inherited rule for `__groupname__` use: [subs="quotes"] ---- group.__groupname__= ---- `__sched__` must be one of: *`f`* for FIFO, *`b`* for batch, *`r`* for round robin, *`o`* for other, *`*`* do not change. `__affinity__` is CPU affinity in hexadecimal. Use `*` for no change. `__prio__` scheduling priority (see `chrt -m`). `__regex__` is Python regular expression. It is matched against the output of: [subs="quotes"] ---- ps -eo cmd ---- Any given process name may match more than one group. In such a case, the priority and scheduling policy are taken from the last matching `__regex__`. .Setting scheduling policy and priorities to kernel threads and watchdog ==== ---- [scheduler] group.kthreads=0:*:1:*:\[.*\]$ group.watchdog=0:f:99:*:\[watchdog.*\] ---- ==== The scheduler plug-in uses perf event loop to catch newly created processes. By default it listens to `perf.RECORD_COMM` and `perf.RECORD_EXIT` events. By setting [option]`perf_process_fork` option to `true`, `perf.RECORD_FORK` events will be also listened to. In other words, child processes created by the `fork()` system call will be processed. Since child processes inherit CPU affinity from their parents, the scheduler plug-in usually does not need to explicitly process these events. As processing perf events can pose a significant CPU overhead, the [option]`perf_process_fork` option parameter is set to `false` by default. Due to this, child processes are not processed by the scheduler plug-in. The CPU overhead of the scheduler plugin can be mitigated by using the scheduler [option]`runtime` option and setting it to `0`. This will completely disable the dynamic scheduler functionality and the perf events will not be monitored and acted upon. The disadvantage ot this approach is the procees/thread tuning will be done only at profile application. .Disabling the scheduler dynamic functionality ==== ---- [scheduler] runtime=0 isolated_cores=1,3 ---- ==== NOTE: For perf events, memory mapped buffer is used. Under heavy load the buffer may overflow. In such cases the `scheduler` plug-in may start missing events and failing to process some newly created processes. Increasing the buffer size may help. The buffer size can be set with the [option]`perf_mmap_pages` option. The value of this parameter has to expressed in powers of 2. If it is not the power of 2, the nearest higher power of 2 value is calculated from it and this calculated value used. If the [option]`perf_mmap_pages` option is omitted, the default kernel value is used. The scheduler plug-in supports process/thread confinement using cgroups v1. [option]`cgroup_mount_point` option specifies the path to mount the cgroup filesystem or where *TuneD* expects it to be mounted. If unset, `/sys/fs/cgroup/cpuset` is expected. If [option]`cgroup_groups_init` option is set to `1` *TuneD* will create (and remove) all cgroups defined with the `cgroup*` options. This is the default behavior. If it is set to `0` the cgroups need to be preset by other means. If [option]`cgroup_mount_point_init` option is set to `1`, *TuneD* will create (and remove) the cgroup mountpoint. It implies `cgroup_groups_init = 1`. If set to `0` the cgroups mount point needs to be preset by other means. This is the default behavior. The [option]`cgroup_for_isolated_cores` option is the cgroup name used for the [option]`isolated_cores` option functionality. For example, if a system has 4 CPUs, `isolated_cores=1` means that all processes/threads will be moved to CPUs 0,2-3. The scheduler plug-in will isolate the specified core by writing the calculated CPU affinity to the `cpuset.cpus` control file of the specified cgroup and move all the matching processes/threads to this group. If this option is unset, classic cpuset affinity using `sched_setaffinity()` will be used. [option]`cgroup.__cgroup_name__` option defines affinities for arbitrary cgroups. Even hierarchic cgroups can be used, but the hieararchy needs to be specified in the correct order. Also *TuneD* does not do any sanity checks here, with the exception that it forces the cgroup to be under [option]`cgroup_mount_point`. The syntax of the scheduler option starting with `group.` has been augmented to use `cgroup.__cgroup_name__` instead of the hexadecimal `__affinity__`. The matching processes will be moved to the cgroup `__cgroup_name__`. It is also possible to use cgroups which have not been defined by the [option]`cgroup.` option as described above, i.e. cgroups not managed by *TuneD*. All cgroup names are sanitized by replacing all all dots (`.`) with slashes (`/`). This is to prevent the plug-in from writing outside [option]`cgroup_mount_point`. .Using cgroups v1 with the scheduler plug-in ==== ---- [scheduler] cgroup_mount_point=/sys/fs/cgroup/cpuset cgroup_mount_point_init=1 cgroup_groups_init=1 cgroup_for_isolated_cores=group cgroup.group1=2 cgroup.group2=0,2 group.ksoftirqd=0:f:2:cgroup.group1:ksoftirqd.* ps_blacklist=ksoftirqd.*;rcuc.*;rcub.*;ktimersoftd.* isolated_cores=1 ---- Cgroup `group1` has the affinity set to CPU 2 and the cgroup `group2` to CPUs 0,2. Given a 4 CPU setup, the [option]`isolated_cores=1` option causes all processes/threads to be moved to CPU cores 0,2-3. Processes/threads that are blacklisted by the [option]`ps_blacklist` regular expression will not be moved. The scheduler plug-in will isolate the specified core by writing the CPU affinity 0,2-3 to the `cpuset.cpus` control file of the `group` and move all the matching processes/threads to this cgroup. ==== Option [option]`cgroup_ps_blacklist` allows excluding processes which belong to the blacklisted cgroups. The regular expression specified by this option is matched against cgroup hierarchies from `/proc/PID/cgroups`. Cgroups v1 hierarchies from `/proc/PID/cgroups` are separated by commas ',' prior to regular expression matching. The following is an example of content against which the regular expression is matched against: `10:hugetlb:/,9:perf_event:/,8:blkio:/` Multiple regular expressions can be separated by semicolon ';'. The semicolon represents a logical 'or' operator. .Cgroup-based exclusion of processes from the scheduler ==== ---- [scheduler] isolated_cores=1 cgroup_ps_blacklist=:/daemons\b ---- The scheduler plug-in will move all processes away from core 1 except processes which belong to cgroup '/daemons'. The '\b' is a regular expression metacharacter that matches a word boundary. ---- [scheduler] isolated_cores=1 cgroup_ps_blacklist=\b8:blkio: ---- The scheduler plug-in will exclude all processes which belong to a cgroup with hierarchy-ID 8 and controller-list blkio. ==== Kernels 5.13 and newer moved some `sched_` and `numa_balancing_` kernel run-time parameters from `/proc/sys/kernel`, managed by the `sysctl` utility, to `debugfs`, typically mounted under `/sys/kernel/debug`. TuneD provides an abstraction mechanism for the following parameters via the scheduler plug-in: [option]`sched_min_granularity_ns`, [option]`sched_latency_ns`, [option]`sched_wakeup_granularity_ns`, [option]`sched_tunable_scaling`, [option]`sched_migration_cost_ns`, [option]`sched_nr_migrate`, [option]`numa_balancing_scan_delay_ms`, [option]`numa_balancing_scan_period_min_ms`, [option]`numa_balancing_scan_period_max_ms` and [option]`numa_balancing_scan_size_mb`. Moreover in kernel 6.6 and newer support for the `sched_wakeup_granularity_ns` and `sched_latency_ns` were removed. The `sched_min_granularity_ns` was renamed to `sched_base_slice_ns`. Based on the kernel used, TuneD will write the specified value to the correct location or ignore it. For the compatibility the alias [option]`sched_base_slice_ns` was added, but the [option]`sched_min_granularity_ns` can be still used instead. .Set tasks' "cache hot" value for migration decisions. ==== ---- [scheduler] sched_migration_cost_ns=500000 ---- On the old kernels, this is equivalent to: ---- [sysctl] kernel.sched_migration_cost_ns=500000 ---- that is, value `500000` will be written to `/proc/sys/kernel/sched_migration_cost_ns`. However, on more recent kernels, the value `500000` will be written to `/sys/kernel/debug/sched/migration_cost_ns`. ==== ÚZ base_slice_ns)Úwakeup_granularity_nsÚmin_granularity_nsÚ latency_nscCst d¡d|_dS)Nzupython-perf unavailable, disabling perf support and runtime tuning, you can try to (re)install python(3)-perf packageF)ÚlogÚwarningÚ_perf_availablerrrrÚ _disable_perfÂs zSchedulerPlugin._disable_perfc s*tt|ƒ ||||||||¡d|_tj|_ttjƒ|_ |durh|  tj tj¡|_t|  tj tj¡ƒ|_ tƒ|_d|_i|_d|_d|_d|_d|_d|_zt ¡|_Wn*ttfyÒ| ¡|j ¡|_Yn0|jdd�|_d|_ |jdd�|_!d|_"z t#ƒ|_$Wnt�y$t%ƒ|_$Yn0dS)NTz.*r\r )Z command_nameÚirq)&Úsuperr[rZ_has_dynamic_optionsÚconstsZCFG_DEF_DAEMONÚ_daemonÚintZCFG_DEF_SLEEP_INTERVALÚ_sleep_intervalÚget_boolZ CFG_DAEMONr<ZCFG_SLEEP_INTERVALrrÚ_secure_boot_hintÚ_sched_knob_paths_cacheÚ _ps_whitelistÚ _ps_blacklistÚ_kthread_processÚ_cgroup_ps_blacklist_rerbÚperfZcpu_mapÚ_cpusÚ NameErrorÚAttributeErrorrcZget_cpusZ _storage_keyÚ_scheduler_storage_keyÚ _irq_processÚ_irq_storage_keyÚ_evlistr Ú_scheduler_utilsrX) rZmonitor_repositoryZstorage_factoryZhardware_inventoryZdevice_matcherZdevice_matcher_udevZplugin_instance_factoryZ global_cfgZ variables©Ú __class__rrrÇs@ ÿÿ zSchedulerPlugin.__init__cCsT|dur dSz t|ƒ}Wnty,YdS0|dkr:dStdt t |d¡¡ƒS)Nré)rhÚ ValueErrorÚmathZceilr`)rZ mmap_pagesZmprrrÚ_calc_mmap_pagesîs  z SchedulerPlugin._calc_mmap_pagesc sd|_d|_d|_d|_ˆj ˆji¡ˆ_tˆjƒdkr^t   d¡ˆ  ¡iˆ_ˆj  ˆj¡t ƒˆ_dˆ_dˆ_dˆ_t ‡fdd„|j ¡Dƒ¡ˆ_|j|_ˆj |jd¡}ˆ |¡}|dkrÖt  d|¡d}|durüt|ƒ|krüt   d ||f¡|jD]}ˆj |j|¡|j|<�qˆj |j d d ¡¡d k�rDd|_t  ¡|_!ˆj"�r|j�rˆj#�rzŠt$ %¡|_&t$j't$j(t$j)d d ddd d t$j*t$j+Bd � }|j,ˆj-|j&d�t$ .ˆj-|j&¡|_|j /|¡|du�rà|j 0¡n|jj0|d�Wnˆ 1¡Yn0ˆj#�sd|_dS)NFTrz0recovering scheduling settings from previous runcsJg|]B\}}|dd…dkrt|ƒdkrˆ |dd…¡ˆj |¡f‘qS©Nézcgroup.)ÚlenÚ_sanitize_cgroup_pathÚ _variablesÚexpand)r/Úoptionr rrrÚ s"ÿz2SchedulerPlugin._instance_init..Úperf_mmap_pageszKInvalid 'perf_mmap_pages' value specified: '%s', using default kernel valuezL'perf_mmap_pages' value has to be power of two, specified: '%s', using: '%d'ZruntimerÚ0) ÚtypeZconfigZtaskÚcommÚmmapZfreqZ wakeup_eventsZ watermarkZ sample_type)ZcpusÚthreads)Zpages)2rxZ_has_dynamic_tuningZ_has_static_tuningÚ_runtime_tuningÚ_storager<ruÚ_scheduler_originalr‚r`ÚinfoÚ_restore_ps_affinityÚunsetr6Ú_cgroups_original_affinityrÚ_cgroup_affinity_initializedÚ_cgroupÚ collectionsÚ OrderedDictÚoptionsr8Ú_cgroupsÚ _schedulerr„r…rÚerrorÚstrrrjÚ threadingÚEventÚ _terminatergrbrqZ thread_mapZ_threadsÚevselZ TYPE_SOFTWAREZCOUNT_SW_DUMMYZ SAMPLE_TIDZ SAMPLE_CPUÚopenrrZevlistÚaddrŒrc)rÚinstanceZperf_mmap_pages_rawrˆr0r¡rrrÚ_instance_initúsjÿ ÿ  ÿ    ü   zSchedulerPlugin._instance_initcCs&|jr"|j ¡D]}t |j¡qdSr)rxÚ get_pollfdr-Úcloser1)rr¤ÚfdrrrÚ_instance_cleanup;sz!SchedulerPlugin._instance_cleanupcCs8dtjdddddddddddddddddddddddœS)NFTÚcalcZfalse)Úisolated_coresÚcgroup_mount_pointÚcgroup_mount_point_initÚcgroup_groups_initÚcgroup_for_isolated_coresÚcgroup_ps_blacklistÚ ps_whitelistÚ ps_blacklistÚkthread_processÚ irq_processÚdefault_irq_smp_affinityrˆÚperf_process_forkÚsched_min_granularity_nsÚsched_base_slice_nsÚsched_latency_nsÚsched_wakeup_granularity_nsÚsched_tunable_scalingÚsched_migration_cost_nsÚsched_nr_migrateÚnuma_balancing_scan_delay_msÚ!numa_balancing_scan_period_min_msÚ!numa_balancing_scan_period_max_msÚnuma_balancing_scan_size_mb)rfZDEF_CGROUP_MOUNT_POINT)ÚclsrrrÚ_get_config_options@s2èz#SchedulerPlugin._get_config_optionscCs|durt|ƒ dd¡SdS)NÚ.ú/)r�Úreplacerrrrrƒ]sz%SchedulerPlugin._sanitize_cgroup_pathcCs>t|tjƒs|}t |¡}t |¡}| |¡r:d|d}|S)Nú[ú])Ú isinstanceÚprocfsÚprocessZprocess_cmdlineÚ _is_kthread)rrËrBr rrrÚ _get_cmdlineas     zSchedulerPlugin._get_cmdlinec CsÖt ¡}| ¡i}| ¡D]´}z^|js6| |¡r6Wq| |¡}|d}|||<d|vr||d ¡D]}| |¡}|||<qdWqtt fyÎ}z4|j t j ksª|j t j kr¸WYd}~qn‚WYd}~qd}~00q|S)NrBr�) rÊÚpidstatsÚreload_threadsr:rorÌrÍÚkeysÚOSErrorÚIOErrorÚerrnoÚENOENTÚESRCH)rÚpsÚ processesÚprocrrBÚerrrÚ get_processesks*     ÿzSchedulerPlugin.get_processescCs@|j |¡}|j |¡}|j |¡}t d|||f¡||fS)Nz8Read scheduler policy '%s' and priority '%d' of PID '%d')ryrCr?rQr`Údebug)rrBr Ú sched_strr rrrÚ_get_rt†s   ÿzSchedulerPlugin._get_rtc Cs|j |¡}t d|||f¡zB|j |¡}|j |¡}||ksJ||kr`t d||||f¡Wn6ttfy˜}zt d|¡WYd}~n d}~00z|j  |||¡Wnbttf�y}zDt |dƒrê|j t j krêt d|¡nt d||f¡WYd}~n d}~00dS)NzBSetting scheduler policy to '%s' and priority to '%d' of PID '%d'.z9Priority for %s must be in range %d - %d. '%d' was given.z(Failed to get allowed priority range: %srÓzAFailed to set scheduling parameters of PID %d, the task vanished.z1Failed to set scheduling parameters of PID %d: %s) ryr?r`rÛrTrVrœÚ SystemErrorrÑrIÚhasattrrÓrÕ)rrBrGrHrÜZprio_minZprio_maxrÙrrrÚ_set_rtŽs6 ÿ  ÿÿ ÿÿÿzSchedulerPlugin._set_rtcCs|ddtjj@dkS)NÚstatÚflagsr)rÊZpidstatZ PF_KTHREAD©rrËrrrr̪szSchedulerPlugin._is_kthreadcCs&|jdkrdSt |j| |¡¡duS)Nr\F)rpÚreÚsearchÚ_get_stat_cgrouprãrrrÚ_process_in_blacklisted_cgroup­s z.SchedulerPlugin._process_in_blacklisted_cgroupc Csˆ|j}zÎ|dddkr,t d|¡WdS| |¡rJt d|¡WdS|d ¡r„| |¡rpt d|¡nt d|¡WdSt d || |¡f¡t d ||  |¡f¡t d |t |j   |¡ƒf¡Wn®t tf�yH}zV|jtjk�s|jtjk�r"t d |¡WYd}~dSt d ||f¡WYd}~nDd}~0ttf�y‚}zt d ||f¡WYd}~n d}~00dS)NráÚstateÚZz9Affinity of zombie task with PID %d could not be changed.Tz`Affinity of task with PID %d could not be changed, the task was moved into a blacklisted cgroup.z[Affinity of kernel thread with PID %d cannot be changed, the task's affinity mask is fixed.zRAffinity of task with PID %d cannot be changed, the task's affinity mask is fixed.zTask %d cmdline: %szTask %d cgroup: %szTask %d affinity: %sz6Failed to get task info for PID %d, the task vanished.z&Failed to get task info for PID %d: %sF)rBr`rÛrçZis_bound_to_cpurÌrar‘rÍræÚlistryrKrÑrÒrÓrÔrÕrœrtÚKeyError)rrËrBrÙrrrÚ_ignore_set_affinity_error´sLÿ ÿ  ÿÿ"ÿÿÿz*SchedulerPlugin._ignore_set_affinity_errorcCsZz|j|}Wn&ty4t|jƒ}||j|<Yn0|jdurV|jdurV||_||_dSr)r�rërrr r )rrBr r ÚparamsrrrÚ_store_orig_process_rtÖs  z&SchedulerPlugin._store_orig_process_rtc CsÎd}|dur|dur|Sz:| |¡\}}|dur4|}| |||¡| |||¡WnvttfyÈ}zZt|dƒr¢|jtjkr¢t  d|¡||j vrœ|j |=d}nt  d||f¡WYd}~n d}~00|S)NTrÓz=Failed to read scheduler policy of PID %d, the task vanished.FzcRefusing to set scheduler and priority of PID %d, reading original scheduling parameters failed: %s) rÝràrîrÞrÑrßrÓrÕr`rÛr�rœ)rrBrGrHÚcontZ prev_schedZ prev_priorÙrrrÚ_tune_process_rtàs*ÿ ÿz SchedulerPlugin._tune_process_rtcCst|ƒdd…dkSr€)r�)rr rrrÚ_is_cgroup_affinityösz#SchedulerPlugin._is_cgroup_affinityFcCs`z|j|}Wn&ty4t|jƒ}||j|<Yn0|jdur\|jdur\|rV||_n||_dSr)r�rërrr r )rrBr Ú is_cgrouprírrrÚ_store_orig_process_affinityùs  z,SchedulerPlugin._store_orig_process_affinityc Csp|jjdtjt|ƒdfdd� d¡D]D}z,| d¡ddd…}|dkrN|nd WStyhYq&0q&d S) Nú%s/%s/%sr T©Úno_errorÚ z:cpuset:rr\rÅ)rÚ read_filerfZPROCFS_MOUNT_POINTr�ÚsplitÚ IndexError)rrBÚlr rrrÚ_get_cgroup_affinitys* z$SchedulerPlugin._get_cgroup_affinitycCsB| |¡}|j}|dkr$d||f}|jjd|t|ƒdd�dS)NrÅú%s/%sz%s/tasksTrõ)rƒÚ_cgroup_mount_pointrÚ write_to_filer�)rrBr ÚpathrrrÚ _set_cgroups   zSchedulerPlugin._set_cgroupcCs,|dd…}t|tƒ o"t|ƒdk}||fS)Nr�r)rÉrêr‚)rr ròrrrÚ_parse_cgroup_affinitys z&SchedulerPlugin._parse_cgroup_affinityc Csðd}|dur|Szd| |¡\}}|r<| |¡}| ||¡n(| |¡}|rX| |||¡}| ||¡| |||¡Wnvttfyê}zZt |dƒrÄ|j t j krÄt   d|¡||jvr¾|j|=d}nt  d||f¡WYd}~n d}~00|S)NTrÓz5Failed to read affinity of PID %d, the task vanished.FzLRefusing to set CPU affinity of PID %d, reading original affinity failed: %s)rrürÚ _get_affinityÚ_get_intersect_affinityÚ _set_affinityrórÞrÑrßrÓrÕr`rÛr�rœ) rrBr Ú intersectrïròr Ú prev_affinityrÙrrrÚ_tune_process_affinity s<  þ ÿÿ ÿz&SchedulerPlugin._tune_process_affinitycCsD| |||¡}|sdS| ||¡}|r0||jvr4dS||j|_dSr)rðrr�r )rrBrrGrHr rïrrrÚ _tune_process?s zSchedulerPlugin._tune_processcCsf|j |¡}|dur.|dkr.t d|¡dSz t|ƒ}Wn"ty\t d|¡YdS0||fS)Nrz>Invalid scheduler: %s. Scheduler and priority will be ignored.)NNz=Invalid priority: %s. Scheduler and priority will be ignored.)ryr>r`rœrhr})rr=Z str_priorityr r rrrÚ_convert_sched_paramsHs ÿ  ÿz%SchedulerPlugin._convert_sched_paramscCsD|dkrd}n2| |¡r|}n"|j |¡}|s@t d|¡d}|S)Nrz)Invalid affinity: %s. It will be ignored.)rñrÚ hex2cpulistr`rœ)rZ str_affinityr rrrÚ_convert_affinityWs  ÿz!SchedulerPlugin._convert_affinitycCs6|\}}}}}| ||¡\}}| |¡}|||||fSr)r r )rÚvalsÚ rule_prior r r ÚregexrrrÚ_convert_sched_cfgds ÿ z"SchedulerPlugin._convert_sched_cfgc Csšd|j|f}zt |tj¡Wn6tyV}zt d||f¡WYd}~n d}~00|jj d|df|jj d|jdfdd�dd�s–t d|¡dS)Nrýz Unable to create cgroup '%s': %sz cpuset.memsTrõz3Unable to initialize 'cpuset.mems ' for cgroup '%s') rþr-ÚmkdirrfÚDEF_CGROUP_MODErÑr`rœrrÿrø)rr rrÙrrrÚ_cgroup_create_groupks(þz$SchedulerPlugin._cgroup_create_groupcCs<|jdur"|j|jvr"| |j¡|jD]}| |¡q(dSr)r–ršr©rÚcgrrrÚ_cgroup_initialize_groupsvs  z)SchedulerPlugin._cgroup_initialize_groupsc CsŽt d¡zt |jtj¡Wn2tyP}zt d|¡WYd}~n d}~00|j   dddddd|jg¡\}}|dkrŠt d |j¡dS) NzInitializing cgroups settingsz'Unable to create cgroup mount point: %sZmountz-tr z-oZcpusetrzUnable to mount '%s') r`rÛr-ÚmakedirsrþrfrrÑrœrÚexecute)rrÙÚretÚoutrrrÚ_cgroup_initialize|s $ z"SchedulerPlugin._cgroup_initializec CsJzt |¡Wn6tyD}zt d||f¡WYd}~n d}~00dS)Nz#Unable to remove directory '%s': %s)r-ÚrmdirrÑr`rœ)rr rÙrrrÚ _remove_dir†szSchedulerPlugin._remove_dircCsTt|jƒD]}| d|j|f¡q |jdurP|j|jvrP| d|j|jf¡dS)Nrý)Úreversedršrrþr–rrrrÚ_cgroup_finalize_groupsŒsz'SchedulerPlugin._cgroup_finalize_groupscCslt d¡|j d|jg¡\}}|dkr.cs6g|].\}}t d|¡rt|ƒdkr|ˆ |¡f‘qS)zgroup\.é)räÚmatchr‚r)r/r†r rrrr‡×s  ýcSs |ddS)Nrrr)Z option_valsrrrÚÛr3z8SchedulerPlugin._instance_apply_static..)Úkeyz(error compiling regular expression: '%s'cs(g|] \}}t ˆ|¡dur||f‘qSr)rärå©r/rBr)r(rrr‡ær3c s$g|]\}}||ˆˆˆˆˆff‘qSrrr4)r r†r rr rrr‡èsÿz(?.ú(?.ra)rJrärùr�rmrbrrrrm“s zSchedulerPlugin._ps_whitelistr²cCs:|rdS|r6|dur6d dd„t dt|ƒ¡Dƒ¡|_dS)Nr^cSsg|] }d|‘qSr_rr`rrrr‡¡r3z1SchedulerPlugin._ps_blacklist..ra)rJrärùr�rnrbrrrrn›s zSchedulerPlugin._ps_blacklistr³cCs*|rdS|r&|dur&|j |¡dk|_dS©Nr,)rrjrorbrrrro£s z SchedulerPlugin._kthread_processr´cCs*|rdS|r&|dur&|j |¡dk|_dSrf)rrjrvrbrrrrv«s zSchedulerPlugin._irq_processrµcCs6|rdS|r2|dur2|dvr$||_n|j |¡|_dS)N)rªÚignore)Ú_default_irq_smp_affinity_valuerÚcpulist_unpackrbrrrÚ_default_irq_smp_affinity³s  z)SchedulerPlugin._default_irq_smp_affinityr¶cCs*|rdS|r&|dur&|j |¡dk|_dSrf)rrjr[rbrrrÚ_perf_process_fork¾s z"SchedulerPlugin._perf_process_forkcCs"|j |¡}t d||f¡|S)NzRead affinity '%s' of PID %d)ryrKr`rÛ)rrBÚresrrrrÊs zSchedulerPlugin._get_affinityc Cs–t |¡}| |¡r&t d|¡dSt d||f¡z|j ||¡WnFttfy�}z*|  |¡s|t  d|||f¡WYd}~n d}~00dS)NzMNot setting CPU affinity of PID %d, the task belongs to a blacklisted cgroup.z'Setting CPU affinity of PID %d to '%s'.z,Failed to set affinity of PID %d to '%s': %s) rÊrËrçr`rÛryrNrÞrÑrìrœ)rrBr rËrÙrrrrÏs   ÿzSchedulerPlugin._set_affinitycCs"t|ƒ t|ƒ¡}|rt|ƒS|Sr)r?Ú intersectionrê)rZ affinity1Z affinity2Z affinity3Zaffrrrrßsz'SchedulerPlugin._get_intersect_affinityc sD|}ˆjs‡fdd„|Dƒ}‡fdd„|Dƒ}ˆjdkrJ‡fdd„|Dƒ}tdd„|Dƒƒ}|D]Þ}zˆ ||¡}Wnnttfyä}zR|jtjks¢|jtjkr²t   d|¡nt   d||f¡WYd}~q`WYd}~n d}~00ˆj ||d d �} | süq`|ˆj v�r|ˆj |_|s`d ||vr`ˆ ||d  ¡|d ¡q`dS) Ncsg|]}ˆ |¡s|‘qSr)rÌr`rrrr‡èr3z9SchedulerPlugin._set_all_obj_affinity..cs(g|] }t ˆjˆ |¡¡dur|‘qSr)rärårmÚ_get_stat_commr`rrrr‡ésÿÿr\cs(g|] }t ˆjˆ |¡¡dur|‘qSr)rärårnrnr`rrrr‡ìsÿÿcSsg|]}|j|f‘qSr)rBr`rrrr‡îr3rSzARefusing to set affinity of PID %d, failed to get its cmdline: %sT)rr�)rornr6rÍrÑrÒrÓrÔrÕr`rÛrœrr�r Ú_set_all_obj_affinityr:) rZobjsr r�ZpslZpsdrBrrÙrïrrrroåsB   ÿÿÿ"ÿ  þz%SchedulerPlugin._set_all_obj_affinityc Cs*z |dWStttfy$YdS0dS)NZcgroupsr\©rÑrÒrë©rr)rrrræs z SchedulerPlugin._get_stat_cgroupc Cs.z|ddWStttfy(YdS0dS)Nrár‹r\rprqrrrrn szSchedulerPlugin._get_stat_commc Csbz&t ¡}| ¡| | ¡|d¡Wn6ttfy\}zt d|¡WYd}~n d}~00dS)NFr-) rÊrÎrÏror:rÑrÒr`rœ)rr rÖrÙrrrÚ_set_ps_affinitysÿz SchedulerPlugin._set_ps_affinityc Csæz`|j |¡}t d||f¡d|}t|dƒ�}| |¡Wdƒn1sR0YWdSttfyà}zdt|dƒrª|j t j krª|sªt d|¡WYd}~dSt  d|||f¡WYd}~d SWYd}~n d}~00dS) Nz&Setting SMP affinity of IRQ %s to '%s'z/proc/irq/%s/smp_affinityÚwrrÓz/Setting SMP affinity of IRQ %s is not supportedéþÿÿÿz0Failed to set SMP affinity of IRQ %s to '%s': %séÿÿÿÿ) rÚ cpulist2hexr`rÛr¢ÚwriterÑrÒrßrÓZEIOrœ)rrdr Z restoringÚ affinity_hexÚfilenamer&rÙrrrÚ_set_irq_affinitys* ÿ (ÿÿÿz!SchedulerPlugin._set_irq_affinityc Cs’zR|j |¡}t d|¡tddƒ�}| |¡Wdƒn1sF0YWn:ttfyŒ}zt d||f¡WYd}~n d}~00dS)Nz(Setting default SMP IRQ affinity to '%s'ú/proc/irq/default_smp_affinityrsz2Failed to set default SMP IRQ affinity to '%s': %s) rrvr`rÛr¢rwrÑrÒrœ)rr rxr&rÙrrrÚ_set_default_irq_affinity6s ÿ ,ÿz)SchedulerPlugin._set_default_irq_affinityc Cstƒ}t ¡}| ¡D]”}z"||d}t d||f¡WntyRYqYn0| |||¡}t|ƒt|ƒkrtq|  ||d¡}|dkr–||j |<q|dkr|j   |¡q|j  d¡}|j  |¡}|jdkrÞ| |||¡}n|jdkrî|j}|jdk�r | |¡||_|j |j|¡dS) Nr zRead affinity of IRQ '%s': '%s'Frrtr{rªrg)rrÊÚ interruptsrÐr`rÛrërr?rzrrÚappendrrør rhr|rr�rw) rr Ú irq_originalrrdrrrlZprev_affinity_hexrrrÚ_set_all_irq_affinityAs8  ÿ          z%SchedulerPlugin._set_all_irq_affinitycCsj|j |jd¡}|durdS|j ¡D]\}}| ||d¡q&|jdkrX|j}| |¡|j  |j¡dS)NTrg) r�r<rwrr8rzrhrr|r“)rrrdr rrrÚ_restore_all_irq_affinity`s  z)SchedulerPlugin._restore_all_irq_affinitycCsFt|ƒ t|ƒ¡}|r,t tj||f¡nt tj|||f¡|Sr)r?Úissubsetr`r‘rfrLrœrM)rÚirq_descriptionÚcorrect_affinityrNrlrrrÚ_verify_irq_affinityksÿÿÿz$SchedulerPlugin._verify_irq_affinityc Csî|j |jd¡}t ¡}d}| ¡D]€}||jvrP|rPd|}t t j |¡q$z<||d}t  d||f¡d|} |  | ||¡sŠd}Wq$t y¢Yq$Yq$0q$|j d¡} |j | ¡}|jdkrê|  d ||jd krÞ|n|j¡sêd}|S) NTz-IRQ %s does not support changing SMP affinityr z#Read SMP affinity of IRQ '%s': '%s'zSMP affinity of IRQ %sFr{rgzdefault IRQ SMP affinityrª)r�r<rwrÊr}rÐrr`r‘rfZ STR_VERIFY_PROFILE_VALUE_MISSINGrÛr…rërrør rh) rr„rRrrrlrdÚ descriptionrNrƒZcurrent_affinity_hexrrrÚ_verify_all_irq_affinityws@  ÿý  ÿ þz(SchedulerPlugin._verify_all_irq_affinityr«é )r]r c Csðd}d|_|durrt|j |¡ƒ}t|jƒ}| |¡rRt||ƒ}|j |¡|_n |j |j¡} t  d|| f¡|sz|r†|dur†dS|r |j rœ|  ||¡SdS|rÞ|j r¾|  ¡d|j } n|} | | ¡|j rì| |¡n|j rì| ¡dS)NzJInvalid isolated_cores specified, '%s' does not match available cores '%s'Tz cgroup.%s)rr?rrirrr‚rêrKr`rœrvr‡r–r*rrr€r�) rrcrrdrRr¤r ÚisolatedZpresentZstr_cpusZ ps_affinityrrrÚ_isolated_cores–s:   ÿ    zSchedulerPlugin._isolated_corescCs,|dkrd||f}nd|||f}d|S)Nr\rýrôz/sys/kernel/debug/%sr)rÚprefixÚ namespaceÚknobrrrrÚ_sched_assembly_path»sz$SchedulerPlugin._sched_assembly_pathcCs<|}|dkr8|j |¡}|dur8|r4| |||¡}nd}|S)NrGr\)Ú_dict_sched_knob_mapr<rŽ)rrr‹rŒr�ZlpathZlknobrrrÚ_sched_assembly_path2Ãs z%SchedulerPlugin._sched_assembly_path2cCs’d|||f}|j |¡}|s&|dkr*|Sd||f}tj |¡s„| |||¡}tj |¡sl| ||||¡}|dkr„|jdur„d|_||j|<|S)Nz%s_%s_%sr\z/proc/sys/kernel/%s_%sT)rlr<r-rÚexistsrŽr�rk)rr‹rŒr�r3rrrrÚ_get_sched_knob_pathÎs      z$SchedulerPlugin._get_sched_knob_pathcCsZd}| |||¡}|dkrV|jj|dd�}|durVt d|¡|jrVt d¡d|_|S)Nr\)r&zError reading '%s'zUThis may not work with Secure Boot or kernel_lockdown (this hint is logged only once)F)r’rrør`rœrk)rr‹rŒr�rFrrrrÚ_get_sched_knobÞs zSchedulerPlugin._get_sched_knobcCsh|dur dS| |||¡}|s0t d|¡dS|sd|jj|||rJtjgndd�sdt d||f¡|S)Nz(knob '%s' ignored, unsupported by kernelFrõz Error writing value '%s' to '%s')r’r`rÛrrÿrÓrÔrœ)rr‹rŒr�rÚsimÚremoverrrrÚ_set_sched_knobês ÿzSchedulerPlugin._set_sched_knobr·cCs| ddd¡S©Nr\rGr^©r“©rr¤rrrÚ_get_sched_min_granularity_ns÷sz-SchedulerPlugin._get_sched_min_granularity_nscCs| ddd|||¡Sr—©r–©rrr¤r”r•rrrÚ_set_sched_min_granularity_nsûsz-SchedulerPlugin._set_sched_min_granularity_nsr¸cCs | |¡Sr)ršr™rrrÚ_get_sched_base_slice_nsÿsz(SchedulerPlugin._get_sched_base_slice_nscCs| ||||¡Sr)r�rœrrrÚ_set_sched_base_slice_nssz(SchedulerPlugin._set_sched_base_slice_nsr¹cCs| ddd¡S©Nr\rGr_r˜r™rrrÚ_get_sched_latency_nssz%SchedulerPlugin._get_sched_latency_nscCs| ddd|||¡Sr r›rœrrrÚ_set_sched_latency_ns sz%SchedulerPlugin._set_sched_latency_nsrºcCs| ddd¡S©Nr\rGr]r˜r™rrrÚ _get_sched_wakeup_granularity_nssz0SchedulerPlugin._get_sched_wakeup_granularity_nscCs| ddd|||¡Sr£r›rœrrrÚ _set_sched_wakeup_granularity_nssz0SchedulerPlugin._set_sched_wakeup_granularity_nsr»cCs| ddd¡S©Nr\rGZtunable_scalingr˜r™rrrÚ_get_sched_tunable_scalingsz*SchedulerPlugin._get_sched_tunable_scalingcCs| ddd|||¡Sr¦r›rœrrrÚ_set_sched_tunable_scalingsz*SchedulerPlugin._set_sched_tunable_scalingr¼cCs| ddd¡S©Nr\rGZmigration_cost_nsr˜r™rrrÚ_get_sched_migration_cost_nssz,SchedulerPlugin._get_sched_migration_cost_nscCs| ddd|||¡Sr©r›rœrrrÚ_set_sched_migration_cost_ns#sz,SchedulerPlugin._set_sched_migration_cost_nsr½cCs| ddd¡S©Nr\rGZ nr_migrater˜r™rrrÚ_get_sched_nr_migrate'sz%SchedulerPlugin._get_sched_nr_migratecCs| ddd|||¡Sr¬r›rœrrrÚ_set_sched_nr_migrate+sz%SchedulerPlugin._set_sched_nr_migrater¾cCs| ddd¡S©NrGÚnuma_balancingZ scan_delay_msr˜r™rrrÚ!_get_numa_balancing_scan_delay_ms/sz1SchedulerPlugin._get_numa_balancing_scan_delay_mscCs| ddd|||¡Sr¯r›rœrrrÚ!_set_numa_balancing_scan_delay_ms3sz1SchedulerPlugin._set_numa_balancing_scan_delay_msr¿cCs| ddd¡S©NrGr°Zscan_period_min_msr˜r™rrrÚ&_get_numa_balancing_scan_period_min_ms7sz6SchedulerPlugin._get_numa_balancing_scan_period_min_mscCs| ddd|||¡Sr³r›rœrrrÚ&_set_numa_balancing_scan_period_min_ms;sz6SchedulerPlugin._set_numa_balancing_scan_period_min_msrÀcCs| ddd¡S©NrGr°Zscan_period_max_msr˜r™rrrÚ&_get_numa_balancing_scan_period_max_ms?sz6SchedulerPlugin._get_numa_balancing_scan_period_max_mscCs| ddd|||¡Sr¶r›rœrrrÚ&_set_numa_balancing_scan_period_max_msCsz6SchedulerPlugin._set_numa_balancing_scan_period_max_msrÁcCs| ddd¡S©NrGr°Z scan_size_mbr˜r™rrrÚ _get_numa_balancing_scan_size_mbGsz0SchedulerPlugin._get_numa_balancing_scan_size_mbcCs| ddd|||¡Sr¹r›rœrrrÚ _set_numa_balancing_scan_size_mbKsz0SchedulerPlugin._set_numa_balancing_scan_size_mb)F)F)F)F)F)jrrrrWr�rcrrr¥r©Ú classmethodrÃrƒrÍrÚrÝràrÌrçrìrîrðrñrórürrrr r r rrrrrrr"r)r*r+r9r’rGrHrfZ ROLLBACK_SOFTrIrOrPrQrUrVrAZcommand_customrermrnrorvrjrkrrrrorærnrrrzr|r€r�r…r‡rŠrŽr�r’r“r–Z command_getršZ command_setr�ržrŸr¡r¢r¤r¥r§r¨rªr«r­r®r±r²r´rµr·r¸rºr»Ú __classcell__rrrzrr[Žs0ý ' A  "           <                 "      $                       r[)!r\rZ decoratorsZ tuned.logsZtunedräÚ subprocessržrqÚ ImportErrorrWZ tuned.constsrfrÊZtuned.utils.commandsrrÓr-r—r~r!rtrZZlogsr<r`Úobjectrrr rXZPluginr[rrrrÚs6        /