Multi Profile option in TuneD

tuned/plugins/base.py

@@ -95,13 +95,15 @@ def _option_bool(self, value):
 
 	def create_instance(self, name, devices_expression, devices_udev_regex, script_pre, script_post, options):
 		"""Create new instance of the plugin and seize the devices."""
-		if name in self._instances:
-			raise Exception("Plugin instance with name '%s' already exists." % name)
 
 		effective_options = self._get_effective_options(options)
 		instance = self._instance_factory.create(self, name, devices_expression, devices_udev_regex, \
 			script_pre, script_post, effective_options)
-		self._instances[name] = instance
+
+		if name in self._instances:
+			self._instances[name].append(instance)
+		else:
+			self._instances[name] = [instance]
 
 		return instance
 
@@ -112,9 +114,13 @@ def destroy_instance(self, instance):
 		if instance.name not in self._instances:
 			raise Exception("Plugin instance '%s' was already destroyed." % instance)
 
-		instance = self._instances[instance.name]
+		if type(self._instances[instance.name]) == list:
+			instance = self._instances[instance.name][0]
+		else:
+			instance = self._instances[instance.name]
 		self._destroy_instance(instance)
-		del self._instances[instance.name]
+		if self._instances[instance.name] == []:
+			del self._instances[instance.name]
 
 	def initialize_instance(self, instance):
 		"""Initialize an instance."""
@@ -123,7 +129,7 @@ def initialize_instance(self, instance):
 
 	def destroy_instances(self):
 		"""Destroy all instances."""
-		for instance in list(self._instances.values()):
+		for instance in sum(list(self._instances.values()), []):
 			log.debug("destroying instance %s (%s)" % (instance.name, self.name))
 			self._destroy_instance(instance)
 		self._instances.clear()

tuned/plugins/plugin_cpu.py

@@ -218,6 +218,30 @@ def _init_devices(self):
 	def _get_device_objects(self, devices):
 		return [self._hardware_inventory.get_device("cpu", x) for x in devices]
 
+	#method overriding of assign_free_devices method from base.py to use cores option
+	def assign_free_devices(self, instance):
+		log.debug("In over written method %s" % instance.name)
+		if not self._devices_supported:
+			return
+
+		if instance._options["cores"] is not None:
+			cores = set(self._cmd.cpulist_unpack(instance._options["cores"]))
+			cores_list = self._cmd.cpulist2string(cores, "cpu")
+			to_assign = self._get_matching_devices(instance, set(cores_list.split(',')))
+		else:
+			to_assign = self._get_matching_devices(instance, self._free_devices)
+		instance.active = len(to_assign) > 0
+		if not instance.active:
+			log.warn("instance %s: no matching devices available" % instance.name)
+		else:
+			name = instance.name
+			if instance.name != self.name:
+				name += " (%s)" % self.name
+			log.info("instance %s: assigning devices %s" % (name, ", ".join(to_assign)))
+			instance.assigned_devices.update(to_assign) # cannot use |=
+			self._assigned_devices |= to_assign
+			self._free_devices -= to_assign
+
 	@classmethod
 	def _get_config_options(self):
 		return {
@@ -233,6 +257,7 @@ def _get_config_options(self):
 			"no_turbo"             : None,
 			"pm_qos_resume_latency_us": None,
 			"energy_performance_preference" : None,
+                        "cores" : None,
 		}
 
 	def _check_arch(self):
@@ -306,9 +331,10 @@ def _instance_init(self, instance):
 		instance._has_static_tuning = True
 		instance._has_dynamic_tuning = False
 
-		# only the first instance of the plugin can control the latency
-		if list(self._instances.values())[0] == instance:
-			instance._first_instance = True
+		# As below instructions will be same for multiple instances, apply them only once
+		# only one instance of the plugin is enough to control the latency.
+		if list(self._instances.values())[0][0] == instance:
+			instance._has_instance = True
 			try:
 				self._cpu_latency_fd = os.open(consts.PATH_CPU_DMA_LATENCY, os.O_WRONLY)
 			except OSError:
@@ -324,20 +350,21 @@ def _instance_init(self, instance):
 
 			self._check_arch()
 		else:
-			instance._first_instance = False
-			log.info("Latency settings from non-first CPU plugin instance '%s' will be ignored." % instance.name)
+			instance._has_instance = False
 
 		try:
 			instance._first_device = list(instance.assigned_devices)[0]
 		except IndexError:
 			instance._first_device = None
 
 	def _instance_cleanup(self, instance):
-		if instance._first_instance:
+		if instance._has_instance:
 			if self._has_pm_qos:
 				os.close(self._cpu_latency_fd)
 			if instance._load_monitor is not None:
 				self._monitors_repository.delete(instance._load_monitor)
+			# set self._has_instance to False to avoid cleanup multiple times
+			instance._has_instance = False
 
 	def _get_intel_pstate_attr(self, attr):
 		return self._cmd.read_file("/sys/devices/system/cpu/intel_pstate/%s" % attr, None).strip()
@@ -356,9 +383,6 @@ def _getset_intel_pstate_attr(self, attr, value):
 	def _instance_apply_static(self, instance):
 		super(CPULatencyPlugin, self)._instance_apply_static(instance)
 
-		if not instance._first_instance:
-			return
-
 		force_latency_value = self._variables.expand(
 			instance.options["force_latency"])
 		if force_latency_value is not None:
@@ -380,7 +404,7 @@ def _instance_apply_static(self, instance):
 	def _instance_unapply_static(self, instance, rollback = consts.ROLLBACK_SOFT):
 		super(CPULatencyPlugin, self)._instance_unapply_static(instance, rollback)
 
-		if instance._first_instance and self._has_intel_pstate:
+		if self._has_intel_pstate:
 			self._set_intel_pstate_attr("min_perf_pct", self._min_perf_pct_save)
 			self._set_intel_pstate_attr("max_perf_pct", self._max_perf_pct_save)
 			self._set_intel_pstate_attr("no_turbo", self._no_turbo_save)
@@ -389,7 +413,6 @@ def _instance_apply_dynamic(self, instance, device):
 		self._instance_update_dynamic(instance, device)
 
 	def _instance_update_dynamic(self, instance, device):
-		assert(instance._first_instance)
 		if device != instance._first_device:
 			return
 

tuned/profiles/loader.py

@@ -52,7 +52,18 @@ def load(self, profile_names):
 		self._load_profile(profile_names, profiles, processed_files)
 
 		if len(profiles) > 1:
-			final_profile = self._profile_merger.merge(profiles)
+			join_profile = False
+
+			for profile in profiles:
+				for unit_name,unit in profile.units.items():
+					if(profile.units[unit_name].merge_type =='join'):
+                                                join_profile = True
+                                                break
+
+			if join_profile == True:
+				final_profile = self._profile_merger.join(profiles)
+			else:
+				final_profile = self._profile_merger.merge(profiles)
 		else:
 			final_profile = profiles[0]
 
@@ -67,12 +78,21 @@ def load(self, profile_names):
 
 	def _expand_vars_in_devices(self, profile):
 		for unit in profile.units:
-			profile.units[unit].devices = self._variables.expand(profile.units[unit].devices)
+			if type(profile.units[unit]) == list:
+                                for sub_unit in range(len(profile.units[unit])):
+                                        profile.units[unit][sub_unit].devices = self._variables.expand(profile.units[unit][sub_unit].devices)
+			else:
+                                profile.units[unit].devices = self._variables.expand(profile.units[unit].devices)
 
 	def _expand_vars_in_regexes(self, profile):
 		for unit in profile.units:
-			profile.units[unit].cpuinfo_regex = self._variables.expand(profile.units[unit].cpuinfo_regex)
-			profile.units[unit].uname_regex = self._variables.expand(profile.units[unit].uname_regex)
+			if type(profile.units[unit]) == list:
+				for sub_unit in range(len(profile.units[unit])):
+                                        profile.units[unit][sub_unit].cpuinfo_regex = self._variables.expand(profile.units[unit][sub_unit].cpuinfo_regex)
+                                        profile.units[unit][sub_unit].uname_regex = self._variables.expand(profile.units[unit][sub_unit].uname_regex)
+			else:
+                                profile.units[unit].cpuinfo_regex = self._variables.expand(profile.units[unit].cpuinfo_regex)
+                                profile.units[unit].uname_regex = self._variables.expand(profile.units[unit].uname_regex)
 
 	def _load_profile(self, profile_names, profiles, processed_files):
 		for name in profile_names:
@@ -106,12 +126,16 @@ def _load_config_data(self, file_name):
 		config = collections.OrderedDict()
 		dir_name = os.path.dirname(file_name)
 		for section in list(config_obj.sections()):
-			config[section] = collections.OrderedDict()
+			split_section = section.split(':')
+			final_section = split_section[0]
+			config[final_section] = collections.OrderedDict()
+			if len(split_section) > 1:
+                                config[final_section]["merge_type"] = split_section[1]
 			for option in config_obj.options(section):
-				config[section][option] = config_obj.get(section, option, raw=True)
-				config[section][option] = self._expand_profile_dir(dir_name, config[section][option])
-			if config[section].get("script") is not None:
-				script_path = os.path.join(dir_name, config[section]["script"])
-				config[section]["script"] = [os.path.normpath(script_path)]
+				config[final_section][option] = config_obj.get(section, option, raw=True)
+				config[final_section][option] = self._expand_profile_dir(dir_name, config[final_section][option])
+			if config[final_section].get("script") is not None:
+				script_path = os.path.join(dir_name, config[final_section]["script"])
+				config[final_section]["script"] = [os.path.normpath(script_path)]
 
 		return config

tuned/profiles/merger.py

@@ -1,5 +1,7 @@
 import collections
 from functools import reduce
+import tuned.logs
+log = tuned.logs.get()
 
 class Merger(object):
 	"""
@@ -17,6 +19,15 @@ def merge(self, configs):
 		merged_config = reduce(self._merge_two, configs)
 		return merged_config
 
+	def join(self, configs):
+		"""
+                Join multiple configurations and append them to a list. This option can be used to apply multiple
+		profiles accross different set of cores simultaneously. To know more about this merge_type, please refer
+                to this document https://docs.google.com/document/d/1Tb6LygN8aM5pdX7akqNe3Wsn65O3oBkPZBEKEX6tZ_s/edit?usp=sharing.
+                """
+		joined_config = reduce(self._join_two, configs)
+		return joined_config
+
 	def _merge_two(self, profile_a, profile_b):
 		"""
 		Merge two profiles. The configuration of units with matching names are updated with options
@@ -55,3 +66,54 @@ def _merge_two(self, profile_a, profile_b):
 					profile_a.units[unit_name].options.update(unit.options)
 
 		return profile_a
+
+	def _join_two(self, profile_a, profile_b):
+		"""
+                Join two profiles. The configuration of units with matching names are updated based on the merge_type
+                the newer profile. If the merge_type of newer profile is "join", append it to the list of existing units.
+		If the merge_type of the newer_profile is "apply", replace(existing Merge merger mechanism).
+		If the 'replace' options of the newer unit is 'True', all options from the older unit are dropped.
+                """
+
+		if type(list(profile_a.units.values())[0]) != list:
+                        profile_a_modified = collections.OrderedDict([(k,[v]) for k,v in profile_a.units.items()])
+                        profile_a.units.clear()
+                        profile_a.units.update(profile_a_modified)
+
+		for unit_name, unit in list(profile_b.units.items()):
+			if unit.replace or unit_name not in profile_a.units:
+                                profile_a.units[unit_name] = [unit]
+
+			else:
+                                if profile_b.units[unit_name].merge_type == "join":
+                                        profile_a.units[unit_name].append(unit)
+
+                                elif profile_b.units[unit_name].merge_type != "join":
+                                        if len(profile_a.units[unit_name]) > 1:
+                                                log.warn("Cannot replace already joined units.")
+                                        else:
+                                                profile_a.units[unit_name][0].type = unit.type
+                                                profile_a.units[unit_name][0].enabled = unit.enabled
+                                                profile_a.units[unit_name][0].devices = unit.devices
+                                                if unit.devices_udev_regex is not None:
+                                                        profile_a.units[unit_name][0].devices_udev_regex = unit.devices_udev_regex
+                                                if unit.cpuinfo_regex is not None:
+                                                        profile_a.units[unit_name][0].cpuinfo_regex = unit.cpuinfo_regex
+                                                if unit.uname_regex is not None:
+                                                        profile_a.units[unit_name][0].uname_regex = unit.uname_regex
+                                                if unit.script_pre is not None:
+                                                        profile_a.units[unit_name][0].script_pre = unit.script_pre
+                                                if unit.script_post is not None:
+                                                        profile_a.units[unit_name][0].script_post = unit.script_post
+                                                if unit.drop is not None:
+                                                        for option in unit.drop:
+                                                                profile_a.units[unit_name][0].options.pop(option, None)
+                                                        unit.drop = None
+                                                if unit_name == "script" and profile_a.units[unit_name][0].options.get("script", None) is not None:
+                                                        script = profile_a.units[unit_name][0].options.get("script", None)
+                                                        profile_a.units[unit_name][0].options.update(unit.options)
+                                                        profile_a.units[unit_name][0].options["script"] = script + profile_a.units[unit_name][0].options["script"]
+                                                else:
+                                                        profile_a.units[unit_name][0].options.update(unit.options)
+
+		return profile_a

tuned/profiles/unit.py

@@ -7,7 +7,7 @@ class Unit(object):
 	"""
 
 	__slots__ = [ "_name", "_type", "_enabled", "_replace", "_drop", "_devices", "_devices_udev_regex", \
-		"_cpuinfo_regex", "_uname_regex", "_script_pre", "_script_post", "_options" ]
+		"_cpuinfo_regex", "_uname_regex", "_script_pre", "_script_post", "_merge_type", "_options" ]
 
 	def __init__(self, name, config):
 		self._name = name
@@ -23,6 +23,7 @@ def __init__(self, name, config):
 		self._uname_regex = config.pop("uname_regex", None)
 		self._script_pre = config.pop("script_pre", None)
 		self._script_post = config.pop("script_post", None)
+		self._merge_type = config.pop("merge_type", "apply")
 		self._options = collections.OrderedDict(config)
 
 	@property
@@ -105,6 +106,14 @@ def script_post(self):
 	def script_post(self, value):
 		self._script_post = value
 
+	@property
+	def merge_type(self):
+		return self._merge_type
+
+	@merge_type.setter
+	def merge_type(self, value):
+		self._merge_type = value
+
 	@property
 	def options(self):
 		return self._options

tuned/units/manager.py

@@ -63,19 +63,35 @@ def _unit_matches_uname(self, unit):
 	def create(self, instances_config):
 		instance_info_list = []
 		for instance_name, instance_info in list(instances_config.items()):
-			if not instance_info.enabled:
-				log.debug("skipping disabled instance '%s'" % instance_name)
-				continue
-			if not self._unit_matches_cpuinfo(instance_info):
-				log.debug("skipping instance '%s', cpuinfo does not match" % instance_name)
-				continue
-			if not self._unit_matches_uname(instance_info):
-				log.debug("skipping instance '%s', uname does not match" % instance_name)
-				continue
-
-			instance_info.options.setdefault("priority", self._def_instance_priority)
-			instance_info.options["priority"] = int(instance_info.options["priority"])
-			instance_info_list.append(instance_info)
+			if type(instance_info) == list:
+                                for sub_instance in instance_info:
+                                        if not sub_instance.enabled:
+                                                log.debug("skipping disabled instance '%s'" % instance_name)
+                                                continue
+                                        if not self._unit_matches_cpuinfo(sub_instance):
+                                                log.debug("skipping instance '%s', cpuinfo does not match" % instance_name)
+                                                continue
+                                        if not self._unit_matches_uname(sub_instance):
+                                                log.debug("skipping instance '%s', uname does not match" % instance_name)
+                                                continue
+
+                                        sub_instance.options.setdefault("priority", self._def_instance_priority)
+                                        sub_instance.options["priority"] = int(sub_instance.options["priority"])
+                                        instance_info_list.append(sub_instance)
+			else:
+				if not instance_info.enabled:
+					log.debug("skipping disabled instance '%s'" % instance_name)
+					continue
+				if not self._unit_matches_cpuinfo(instance_info):
+					log.debug("skipping instance '%s', cpuinfo does not match" % instance_name)
+					continue
+				if not self._unit_matches_uname(instance_info):
+					log.debug("skipping instance '%s', uname does not match" % instance_name)
+					continue
+
+				instance_info.options.setdefault("priority", self._def_instance_priority)
+				instance_info.options["priority"] = int(instance_info.options["priority"])
+				instance_info_list.append(instance_info)
 
 		instance_info_list.sort(key=lambda x: x.options["priority"])
 		plugins_by_name = collections.OrderedDict()