feat: persistence of dynamic tuning changes

Currently, any changes made to the tuning via the `instance_*` dbus calls
are lost when tuning is stopped by the service, or when the TuneD service
itself is stopped/restarted, or when the service crashes.

This commit:

* implements a sync of Plugin Instances and Profile Units that is currently missing.
  This way, dynamic instances and device assignments are persistent across stop/start
  dbus calls to TuneD.
* calculates a hash of the current profile after loading it from disk
  (after processing all includes, so we have a "flat" representation)
* creates snapshots of the current profile whenever instances or assigned
  devices change. the snapshot includes the hash of the profile as
  it was initially loaded. for each instance it stores the devices that
  are currently attached.
* restores a snapshot found at startup, if the hashes match (i.e. there
  have been no profile switches and no changes to the profile or any of
  its includes on disk)

  snapshots are restored in case of
  - daemon restarts (systemctl restart/stop/start)
  - daemon crashes
  snapshots are NOT restored in case of
  - reboots (snapshots are stored in /var/run)
  - profile changes (snapshots are explicitly deleted when switching profiles,
    even when "switching" to the same/current profile)

Signed-off-by: Adriaan Schmidt <adriaan.schmidt@siemens.com>

Author: adriaan42

tests/unit/profiles/test_profile.py

@@ -46,7 +46,7 @@ def test_sets_options(self):
 			"network" : { "type": "net", "devices": "*" },
 		})
 
-		self.assertIs(type(profile.options), dict)
+		self.assertIs(type(profile.options), collections.OrderedDict)
 		self.assertEqual(profile.options["anything"], 10)
 
 	def test_sets_options_empty(self):

tuned/consts.py

@@ -19,6 +19,7 @@
 DEFAULT_STORAGE_FILE = "/run/tuned/save.pickle"
 USER_PROFILES_DIR = "/etc/tuned/profiles"
 SYSTEM_PROFILES_DIR = "/usr/lib/tuned/profiles"
+PROFILE_SNAPSHOT_FILE = "/run/tuned/profile-snapshot.conf"
 PERSISTENT_STORAGE_DIR = "/var/lib/tuned"
 PLUGIN_MAIN_UNIT_NAME = "main"
 PLUGIN_VARIABLES_UNIT_NAME = "variables"

tuned/daemon/controller.py

@@ -417,6 +417,7 @@ def instance_acquire_devices(self, devices, instance_name, caller = None):
 			rets = "Ignoring devices not handled by any instance '%s'." % str(devs)
 			log.info(rets)
 			return (False, rets)
+		self._daemon.sync_instances()
 		return (True, "OK")
 
 	@exports.export("s", "(bsa(ss))")
@@ -482,6 +483,8 @@ def instance_create(self, plugin_name, instance_name, options, caller = None):
 		"""
 		if caller == "":
 			return (False, "Unauthorized")
+		plugin_name = str(plugin_name)
+		instance_name = str(instance_name)
 		if not self._cmd.is_valid_name(plugin_name):
 			return (False, "Invalid plugin_name")
 		if not self._cmd.is_valid_name(instance_name):
@@ -529,6 +532,7 @@ def instance_create(self, plugin_name, instance_name, options, caller = None):
 					other_instance.name, instance.name))
 				plugin._remove_devices_nocheck(other_instance, devs_moving)
 				plugin._add_devices_nocheck(instance, devs_moving)
+		self._daemon.sync_instances()
 		return (True, "OK")
 
 	@exports.export("s", "(bs)")
@@ -571,4 +575,5 @@ def instance_destroy(self, instance_name, caller = None):
 		for device in devices:
 			# _add_device() will find a suitable plugin instance
 			plugin._add_device(device)
+		self._daemon.sync_instances()
 		return (True, "OK")

tuned/daemon/daemon.py

@@ -123,6 +123,11 @@ def _load_profiles(self, profile_names, manual):
 			self._notify_profile_changed(profile_names, False, errstr)
 			raise TunedException(errstr)
 
+		# restore profile snapshot (if there is one)
+		snapshot = self._profile_loader.restore_snapshot(self._profile)
+		if snapshot is not None:
+			self._profile = snapshot
+
 	def set_profile(self, profile_names, manual):
 		if self.is_running():
 			errstr = "Cannot set profile while the daemon is running."
@@ -156,6 +161,40 @@ def set_all_profiles(self, active_profiles, manual, post_loaded_profile,
 			self._save_active_profile(active_profiles, manual)
 			self._save_post_loaded_profile(post_loaded_profile)
 
+	def sync_instances(self):
+		# NOTE: currently, Controller creates the new instances, and here in Daemon
+		#       we discover what happened, and update the profile accordingly.
+		#       a potentially better approach would be to move some of the logic
+		#       from Controller to Daemon, and create/destroy the instances here,
+		#       and at the same time update the profile.
+
+		# remove all units that don't have an instance
+		instance_names = [i.name for i in self._unit_manager.instances]
+		for unit in list(self._profile.units.keys()):
+			if unit in instance_names:
+				continue
+			log.debug("snapshot sync: removing unit '%s'" % unit)
+			del self._profile.units[unit]
+		# create units for new instances
+		for instance in self._unit_manager.instances:
+			if instance.name in self._profile.units:
+				continue
+			log.debug("snapshot sync: creating unit '%s'" % instance.name)
+			config = {
+				"priority": instance.priority,
+				"type": instance._plugin.name,
+				"enabled": instance.active,
+				"devices": instance.devices_expression,
+				"devices_udev_regex": instance.devices_udev_regex,
+				"script_pre": instance.script_pre,
+				"script_post": instance.script_post,
+			}
+			for k, v in instance.options.items():
+				config[k] = v
+			self._profile.units[instance.name] = self._profile._create_unit(instance.name, config)
+		# create profile snapshot
+		self._profile_loader.create_snapshot(self._profile, self._unit_manager.instances)
+
 	@property
 	def profile(self):
 		return self._profile
@@ -202,6 +241,8 @@ def _thread_code(self):
 		self._save_active_profile(" ".join(self._active_profiles),
 					  self._manual)
 		self._save_post_loaded_profile(self._post_loaded_profile)
+		# trigger a profile snapshot
+		self.sync_instances()
 		self._unit_manager.start_tuning()
 		self._profile_applied.set()
 		log.info("static tuning from profile '%s' applied" % self._profile.name)
@@ -370,6 +411,7 @@ def stop(self, profile_switch = False):
 			return False
 		log.info("stopping tuning")
 		if profile_switch:
+			self._profile_loader.remove_snapshot()
 			self._terminate_profile_switch.set()
 		self._terminate.set()
 		self._thread.join()

tuned/plugins/base.py

@@ -164,23 +164,35 @@ def _get_matching_devices(self, instance, devices):
 			udev_devices = self._device_matcher_udev.match_list(instance.devices_udev_regex, udev_devices)
 			return set([x.sys_name for x in udev_devices])
 
+	def restore_devices(self, instance, devices):
+		if not self._devices_supported:
+			return
+
+		log.debug("Restoring devices of instance %s: %s" % (instance.name, " ".join(devices)))
+		for device in devices:
+			if device not in self._free_devices:
+				continue
+			self._free_devices.remove(device)
+			instance.assigned_devices.add(device)
+			self._assigned_devices.add(device)
+
 	def assign_free_devices(self, instance):
 		if not self._devices_supported:
 			return
 
 		log.debug("assigning devices to instance %s" % instance.name)
 		to_assign = self._get_matching_devices(instance, self._free_devices)
-		instance.active = len(to_assign) > 0
-		if not instance.active:
-			log.warning("instance %s: no matching devices available" % instance.name)
-		else:
+		if len(to_assign) > 0:
 			name = instance.name
 			if instance.name != self.name:
 				name += " (%s)" % self.name
 			log.info("instance %s: assigning devices %s" % (name, ", ".join(to_assign)))
 			instance.assigned_devices.update(to_assign) # cannot use |=
 			self._assigned_devices |= to_assign
 			self._free_devices -= to_assign
+		instance.active = len(instance.assigned_devices) > 0
+		if not instance.active:
+			log.warning("instance %s: no matching devices available" % instance.name)
 
 	def release_devices(self, instance):
 		if not self._devices_supported:

tuned/profiles/loader.py

@@ -24,9 +24,6 @@ def __init__(self, profile_locator, profile_factory, profile_merger, global_conf
 		self._global_config = global_config
 		self._variables = variables
 
-	def _create_profile(self, profile_name, config):
-		return tuned.profiles.profile.Profile(profile_name, config)
-
 	@classmethod
 	def safe_name(cls, profile_name):
 		return re.match(r'^[a-zA-Z0-9_.-]+$', profile_name)
@@ -57,6 +54,7 @@ def load(self, profile_names):
 		# FIXME hack, do all variable expansions in one place
 		self._expand_vars_in_devices(final_profile)
 		self._expand_vars_in_regexes(final_profile)
+		final_profile.calculate_hash()
 		return final_profile
 
 	def _expand_vars_in_devices(self, profile):
@@ -68,6 +66,47 @@ def _expand_vars_in_regexes(self, profile):
 			profile.units[unit].cpuinfo_regex = self._variables.expand(profile.units[unit].cpuinfo_regex)
 			profile.units[unit].uname_regex = self._variables.expand(profile.units[unit].uname_regex)
 
+	def create_snapshot(self, profile, instances):
+		snapshot = profile.snapshot(instances)
+		log.debug("Storing profile snapshot in %s:\n%s" % (consts.PROFILE_SNAPSHOT_FILE, snapshot))
+		with open(consts.PROFILE_SNAPSHOT_FILE, "w") as f:
+			f.write(snapshot)
+
+	def restore_snapshot(self, profile):
+		if profile is None:
+			# When tuning is stopped, we are called with profile==None -> skip
+			return None
+		snapshot = None
+		if os.path.isfile(consts.PROFILE_SNAPSHOT_FILE):
+			log.debug("Found profile snapshot '%s'" % consts.PROFILE_SNAPSHOT_FILE)
+			try:
+				config = self._load_config_data(consts.PROFILE_SNAPSHOT_FILE)
+				snapshot_hash = config.get("main", {}).get("profile_base_hash", None)
+				if snapshot_hash == profile._base_hash:
+					snapshot = self._profile_factory.create("restore", config)
+					snapshot.name = profile.name
+					# the snapshot is created directly (not via the merger),
+					# so extract its [variables] section manually
+					if consts.PLUGIN_VARIABLES_UNIT_NAME in snapshot.units:
+						snapshot.variables.update(snapshot.units[consts.PLUGIN_VARIABLES_UNIT_NAME].options)
+						del snapshot.units[consts.PLUGIN_VARIABLES_UNIT_NAME]
+					self._variables.add_from_cfg(snapshot.variables)
+					self._expand_vars_in_devices(snapshot)
+					self._expand_vars_in_regexes(snapshot)
+					log.info("Restored profile snapshot: %s" % snapshot.name)
+				else:
+					log.debug("Snapshot hash '%s' does not match current base hash '%s'. Not restoring." % (snapshot_hash, profile._base_hash))
+					os.remove(consts.PROFILE_SNAPSHOT_FILE)
+			except InvalidProfileException as e:
+				log.error("Could not process profile snapshot: %s" % e)
+		return snapshot
+
+	def remove_snapshot(self):
+		try:
+			os.remove(consts.PROFILE_SNAPSHOT_FILE)
+		except FileNotFoundError:
+			pass
+
 	def _load_profile(self, profile_names, profiles, processed_files):
 		for name in profile_names:
 			filename = self._profile_locator.get_config(name, processed_files)

tuned/profiles/profile.py

@@ -1,24 +1,27 @@
 import tuned.profiles.unit
 import tuned.consts as consts
 import collections
+import hashlib
+import json
 
 class Profile(object):
 	"""
 	Representation of a tuning profile.
 	"""
 
-	__slots__ = ["_name", "_options", "_variables", "_units"]
+	__slots__ = ["_name", "_options", "_variables", "_units", "_base_hash"]
 
 	def __init__(self, name=None, config={}):
 		self._name = name
 		self._variables = collections.OrderedDict()
 		self._init_options(config)
 		self._init_units(config)
+		self._base_hash = config.get("main", {}).get("profile_base_hash", None)
 
 	def _init_options(self, config):
 		self._options = {}
 		if consts.PLUGIN_MAIN_UNIT_NAME in config:
-			self._options = dict(config[consts.PLUGIN_MAIN_UNIT_NAME])
+			self._options = collections.OrderedDict(config[consts.PLUGIN_MAIN_UNIT_NAME])
 
 	def _init_units(self, config):
 		self._units = collections.OrderedDict()
@@ -30,6 +33,35 @@ def _init_units(self, config):
 	def _create_unit(self, name, config):
 		return tuned.profiles.unit.Unit(name, config)
 
+	def as_ordered_dict(self):
+		"""generate serializable (with json.dumps()) representation for hashing"""
+		profile_dict = collections.OrderedDict()
+		profile_dict["main"] = self.options
+		profile_dict["variables"] = self._variables
+		for name, unit in self._units.items():
+			profile_dict[name] = unit.as_ordered_dict()
+		return profile_dict
+
+	def calculate_hash(self):
+		serialized = json.dumps(self.as_ordered_dict())
+		self._base_hash = hashlib.md5(serialized.encode(), usedforsecurity=False).hexdigest()
+
+	def snapshot(self, instances):
+		"""generate config representation that will re-create the data when read as a profile"""
+		snapshot = "[main]\n"
+		snapshot += "active_profile=%s\n" % self.name
+		snapshot += "profile_base_hash=%s\n" % self._base_hash
+		snapshot += "\n[variables]\n"
+		for key, value in self._variables.items():
+			snapshot += "%s=%s\n" % (key, value)
+		for unit in self.units.values():
+			snapshot += "\n" + unit.snapshot()
+			for instance in instances:
+				if instance.name == unit.name:
+					snapshot += "__devices__=%s\n" % " ".join(instance.assigned_devices | instance.processed_devices)
+					break
+		return snapshot
+
 	@property
 	def name(self):
 		"""

tuned/profiles/unit.py

@@ -27,6 +27,55 @@ def __init__(self, name, config):
 		self._script_post = config.pop("script_post", None)
 		self._options = collections.OrderedDict(config)
 
+	def as_ordered_dict(self):
+		"""generate serializable (with json.dumps()) representation for hashing"""
+		ret = collections.OrderedDict()
+		ret["name"] = self.name
+		ret["priority"] = self.priority
+		ret["type"] = self.type
+		ret["enabled"] = self.enabled
+		ret["replace"] = self.replace
+		ret["drop"] = self.drop
+		ret["devices"] = self.devices
+		ret["devices_udev_regex"] = self.devices_udev_regex
+		ret["cpuinfo_regex"] = self.cpuinfo_regex
+		ret["uname_regex"] = self.uname_regex
+		ret["script_pre"] = self.script_pre
+		ret["script_post"] = self.script_post
+		for k, v in self.options.items():
+			ret[k] = v
+		return ret
+
+	@staticmethod
+	def _snapshot_value(value):
+		"""serialize an option value into a form that round-trips through the profile loader"""
+		# some options (e.g. the script plugin's "script") are stored as lists of
+		# absolute paths; emit them space-joined so they are not rendered as a list repr
+		if isinstance(value, list):
+			return " ".join(str(v) for v in value)
+		return str(value)
+
+	def snapshot(self):
+		"""generate config representation that will re-create the data when read as a profile"""
+		snapshot = "[%s]\n" % self.name
+		snapshot += "priority=%s\n" % self.priority
+		snapshot += "type=%s\n" % self.type
+		snapshot += "enabled=%s\n" % self.enabled
+		snapshot += "devices=%s\n" % self.devices
+		if self.devices_udev_regex is not None:
+			snapshot += "devices_udev_regex=%s\n" % self.devices_udev_regex
+		if self.cpuinfo_regex is not None:
+			snapshot += "cpuinfo_regex=%s\n" % self.cpuinfo_regex
+		if self.uname_regex is not None:
+			snapshot += "uname_regex=%s\n" % self.uname_regex
+		if self.script_pre is not None:
+			snapshot += "script_pre=%s\n" % self._snapshot_value(self.script_pre)
+		if self.script_post is not None:
+			snapshot += "script_post=%s\n" % self._snapshot_value(self.script_post)
+		for k, v in self.options.items():
+			snapshot += "%s=%s\n" % (k, self._snapshot_value(v))
+		return snapshot
+
 	@property
 	def name(self):
 		return self._name

tuned/units/manager.py

@@ -98,17 +98,24 @@ def create(self, instances_config):
 				continue
 
 		instances = []
+		instance_restore_devices = {}
 		for instance_info in instance_info_list:
 			plugin = plugins_by_name[instance_info.type]
 			if plugin is None:
 				continue
 			log.debug("creating '%s' (%s)" % (instance_info.name, instance_info.type))
+			restore = instance_info.options.pop("__devices__", None)
+			if restore:
+				instance_restore_devices[instance_info.name] = restore.split()
 			new_instance = plugin.create_instance(instance_info.name, instance_info.priority, \
 				instance_info.devices, instance_info.devices_udev_regex, \
 				instance_info.script_pre, instance_info.script_post, instance_info.options)
 			instances.append(new_instance)
 		for instance in instances:
 			instance.plugin.init_devices()
+			if instance.name in instance_restore_devices:
+				instance.plugin.restore_devices(instance, instance_restore_devices[instance.name])
+		for instance in instances:
 			instance.plugin.assign_free_devices(instance)
 			instance.plugin.initialize_instance(instance)
 		# At this point we should be able to start the HW events